def open(self):
"""Creates an Exiftool process.
Not recommended, use `with:` context manager instead.
"""
if self._exiftool is not None:
raise Exception("ExifTool was already initialized.")
self._exiftool = ExifTool(executable_=str(self.exiftoolpath))
self._exiftool.start()
def _setup_and_add_path(self, parent_id, path):
with ExifTool() as exif_tool:
try:
self._power_manager.acquire()
self._add_path(parent_id, path, exif_tool)
finally:
self._power_manager.release()
def make_plan(self):
"""Create a mapping to know which input files go where in the output"""
plan = collections.defaultdict(list)
destinations = set()
input_paths = self.get_files()
with ExifTool() as exiftool:
for input_path in tqdm(input_paths,
desc='Reading input',
dynamic_ncols=True):
output_path = self.get_output_path(input_path, exiftool)
if output_path in destinations:
raise Exception(
f'Multiple files have the same destination!\n {input_path}\t→\t{output_path}.'
)
if output_path.is_file():
raise Exception(
f'A file already exists at destination path!\n {input_path}\t→\t{output_path}.'
)
destinations.add(output_path)
plan[output_path.parent].append(
Map(
source=input_path,
destination=output_path,
))
return plan
def run(self):
results = {}
results["magic"] = magic.from_file(self.filepath)
results["mimetype"] = magic.from_file(self.filepath, mime=True)
binary = get_binary(self.job_id)
results["md5"] = hashlib.md5(binary).hexdigest()
results["sha1"] = hashlib.sha1(binary).hexdigest()
results["sha256"] = hashlib.sha256(binary).hexdigest()
results["ssdeep"] = pydeep.hash_file(self.filepath).decode()
try:
with ExifTool(self.exiftool_path) as et:
exif_report = et.execute_json(self.filepath)
if exif_report:
exif_single_report = exif_report[0]
exif_report_cleaned = {
key: value
for key, value in exif_single_report.items()
if not (key.startswith("File") or key.startswith("SourceFile"))
}
# compatibility with the previous version of this analyzer
results["filetype"] = exif_single_report.get("File:FileType", "")
results["exiftool"] = exif_report_cleaned
except Exception as e:
logger.exception(e)
return results
def analyze(self, file_path):
with ExifTool() as et:
metadata = et.get_metadata(file_path)
result = {key:value for key, value in metadata.items() if key not in ['SourceFile', 'File:Directory']}
self.file_details = result
return result
def getMeta(self, files):
et = ExifTool()
et.start()
a = et.get_metadata_batch(files)
et.terminate()
meta = self._removeErrors(a)
return meta
def exiftool_tags(*paths):
with ExifTool() as tool:
tags_list = tool.get_tags_batch([], paths)
tags_dict = {}
for tags in tags_list:
file = tags.pop('SourceFile')
tags_dict[file] = tags
return tags_dict
def create_image_doc(date, flight_name, image_loc):
"""Add a new image to the database"""
with ExifTool() as et:
metadata = et.get_metadata(image_loc)
image_doc = {
"date": date,
"flight_name": flight_name,
"image_loc": image_loc,
"metadata": metadata,
"annotated": False,
}
return image_doc
def _import_original(self, path: str):
d = self.components['original'] = {}
d['accession_path'] = realpath(expanduser(expandvars(normpath(path))))
d['filename'] = basename(d['accession_path'])
fn, ext = splitext(d['filename'])
target_path = join(self.path, 'data', d['filename'])
shutil.copy2(d['accession_path'], target_path)
with ExifTool() as et:
meta = et.get_metadata(target_path)
pprint(meta)
xmp = XMPFiles(file_path=target_path).get_xmp()
pprint(xmp)
self._update(manifests=True)
def get_metadata(filepath):
"""
Get Exchangeable image file format information from an image
.. code:: python
filepath = './IMG_001.jpg'
metadata = get_metadata(filepath)
"""
with ExifTool() as et:
metadata = et.get_metadata(filepath)
return metadata
def extract_info(image_file):
"""Extract necessary data from metadata dictionary."""
image_file = image_file.replace("'", "")
with ExifTool() as et:
metadata = et.get_metadata(image_file)
data = {}
data["field_of_view"] = metadata["Composite:FOV"]
data["camera_yaw"] = metadata["MakerNotes:CameraYaw"]
data["relative_altitude"] = float(metadata["XMP:RelativeAltitude"])
data["img_lat"] = metadata["Composite:GPSLatitude"]
data["img_lon"] = metadata["Composite:GPSLongitude"]
data["img_width"] = metadata["File:ImageWidth"]
data["img_height"] = metadata["File:ImageHeight"]
return data
def _get_tag_bytes(
exiftool: ExifTool, tag: typing.Text, filepath: Path
) -> bytes:
"""Gets the data of a tag in a file in bytes using exiftool.
No batch version is possible, as there is no way to seperate the bytes
output by exiftool.
Parameters:
exiftool: The ExifTool process to use.
tag: The metadata tag to load.
filepath: The path to the file to load.
Returns:
The loaded metadata for the tag in `bytes`.
"""
params = ["-b", f"-{tag}", str(filepath)]
params_as_bytes = map(fsencode, params)
return exiftool.execute(*params_as_bytes)
def get_thermal_batch(
exiftool: ExifTool, filepaths: typing.Iterable[Path]
) -> typing.Iterable[np.ndarray]:
"""Loads the thermal images from multiple FLIR images.
Parameters:
exiftool: The ExifTool process to use.
filepaths: A list of paths to the files to load.
Returns:
A list of thermal data in Celcius as 2-D numpy arrays.
"""
str_paths = [get_str_filepath(filepath) for filepath in filepaths]
metadata = exiftool.get_tags_batch(exif_var_tags.values(), str_paths)
raw_images = [_get_raw_np(exiftool, filepath) for filepath in str_paths]
return [
convert_image(image_metadata, raw_image)
for image_metadata, raw_image in zip(metadata, raw_images)
]
def __init__(self):
self._exiftool = ExifTool()
self._tempfiles = set()
def setUp(self):
self.image_path = Path(
__file__).parent.parent / 'demo/source/APL_082158.NEF'
with ExifTool() as exiftool:
self.info = file_info.FileInfo(self.image_path, exiftool)
pconn = sqlite3.connect(placesDbPath)
pconn.row_factory = sqlite3.Row
pdb = pconn.cursor()
ldb = conn.cursor()
pdb.execute("SELECT COUNT(*) FROM RKPlace")
numPlaces = pdb.fetchone()[0]
print("Found %d places." % numPlaces)
# No images?
if numImages == 0:
sys.exit(0)
if args.exif:
et = ExifTool()
et.start()
index = 0
copied = 0
ignored = 0
stack = []
stack_timestamp = ""
places_freq = dict()
# Iterate over the photos.
for row in db.execute('''
SELECT m.imagePath, m.fileName, v.imageDate AS date, v.imageTimeZoneOffsetSeconds AS offset, v.uuid, v.modelId
FROM RKMaster AS m
INNER JOIN RKVersion AS v ON v.masterId = m.modelId
for row in arows:
uuid=row["uuid"]
albumName=row["name"]
uuIdAlbum[uuid]=albumName;
print("Found %d albums" % len(uuIdAlbum))
if args.verbose:
print("Albums found:", uuIdAlbum)
# No images?
if numImages == 0:
sys.exit(0)
if args.exif:
et = ExifTool();
et.start();
def placeByModelId(modelId):
ldb.execute('''
SELECT placeId
FROM RKPlaceForVersion
WHERE versionId = ?''', (modelId,))
placeIds = ', '.join([str(placeId[0]) for placeId in ldb.fetchall()])
if len(placeIds):
pdb.execute('''
SELECT DISTINCT defaultName AS name
FROM RKPlace
WHERE modelId IN(%s)
class FlirExtractor:
"""Extracts thermal data from FLIR images using ExifTool.
Attributes:
exiftoolpath: The path to the ExifTool executable.
Example:
with FlirExtractor(exiftoolpath="/usr/bin/exiftool") as extractor:
# get a single thermal data point
thermal_data = extractor.get_thermal("./path/to/FLIR.jpg")
# get multiple thermal data
thermal_d_list = extractor.get_thermal_batch(
["./FLIR1.jpg", "./FLIR2.jpg"]
)
"""
exiftoolpath: Optional[Path]
_exiftool: Optional[ExifTool]
def __init__(self, exiftoolpath: Path = exiftool_default_exe):
self.exiftoolpath = exiftoolpath
self._exiftool = None
@property
def exiftool(self) -> ExifTool:
_exiftool = self._exiftool
if _exiftool is None:
raise AttributeError(
"ExifTool was not initialized. "
"Use FlirExtractor in a context manager, e.g. \n"
"with FlirExtractor() as e:\n"
" e.do_magic()")
return _exiftool
def open(self):
"""Creates an Exiftool process.
Not recommended, use `with:` context manager instead.
"""
if self._exiftool is not None:
raise Exception("ExifTool was already initialized.")
self._exiftool = ExifTool(executable_=str(self.exiftoolpath))
self._exiftool.start()
def close(self):
"""Closes the Exiftool process.
Not recommended, use `with:` context manager instead.
"""
if self._exiftool is None:
return # already closed, do nothing
self._exiftool.terminate()
self._exiftool = None
def __enter__(self):
self.open()
return self
def __exit__(self, exception_type, exception_value, traceback):
self.close()
def get_thermal(self, filepath: Path) -> "np.ndarray":
"""Gets a thermal image from a FLIR file.
Parameters:
filepath: The path to the FLIR file.
Returns:
The thermal data in Celcius as a 2-D numpy array.
"""
return get_thermal(self.exiftool, filepath)
def get_thermal_batch(self,
filepaths: Iterable[Path]) -> Iterable["np.ndarray"]:
"""Gets thermal images from a list of FLIR files.
Parameters:
filepaths: The paths to the FLIR files.
Returns:
A list of the thermal data in Celcius as 2-D numpy arrays.
"""
return get_thermal_batch(self.exiftool, filepaths)
"Exiftool executable, eg '/usr/bin/exiftool'. By default, we'll assume exiftool is on the PATH"
)
args = parser.parse_args()
# print(args.files[0])
# print('exiftool', args.exiftool)
files = args.files
# if we're on windows the glob won't be pre-evaluated
if len(files) == 1 and "*" in files[0]:
files = glob.glob(files[0])
exiftool = 'exiftool'
if args.exiftool != None:
exiftool = args.exiftool
with ExifTool(exiftool) as e:
metadatas = e.get_tags_batch(interestingTags, files)
for fileName, metadata in zip(files, metadatas):
with open(fileName, 'rb') as imageFile:
md5 = hashlib.md5()
sha1 = hashlib.sha1()
buffer = imageFile.read()
md5.update(buffer)
sha1.update(buffer)
metadata['md5'] = md5.hexdigest()
metadata['sha1'] = sha1.hexdigest()
print json.dumps(metadatas)
from exiftool import ExifTool
img = "DJI_0063.JPG"
with ExifTool() as et:
metadata = et.get_metadata(img)
def get_metadata(self,img):
with ExifTool() as et:
md = et.get_metadata(img)
return md
def setUp(self):
if not os.path.exists(TestExifManager.staging_dir):
os.mkdir(TestExifManager.staging_dir)
self._exiftool = ExifTool()
class TestExifManager(OPennTestCase):
staging_dir = os.path.join(os.path.dirname(__file__), 'staging')
source_dir = os.path.join(os.path.dirname(__file__), 'data/mscodex1223')
test_image_names = [ 'mscodex1223_wk1_body0001.tif', 'mscodex1223_wk1_body0002.tif', 'mscodex1223_wk1_body0003.tif' ]
test_images = [ os.path.join(source_dir, img) for img in test_image_names ]
staged_images = [ os.path.join(staging_dir, img) for img in test_image_names ]
img_desc = { 'ImageDescription': 'This is the image description' }
img_desc_nl = { 'ImageDescription': """This is the two-line
description""" }
xmp_marked = { 'xmpRights:Marked': 'true' }
# the following has a unicode (c) symbol
dc_rights = { 'dc:Rights': 'The dc rights ©2014' }
tag_dict = { 'Marked': True, 'rights': 'The dc rights ©2014' }
def setUp(self):
if not os.path.exists(TestExifManager.staging_dir):
os.mkdir(TestExifManager.staging_dir)
self._exiftool = ExifTool()
def tearDown(self):
if os.path.exists(TestExifManager.staging_dir):
shutil.rmtree(TestExifManager.staging_dir)
self._exiftool.terminate()
def get_metadata(self,img):
with ExifTool() as et:
md = et.get_metadata(img)
return md
def stage_images(self):
for f in TestExifManager.test_images:
shutil.copy(f, TestExifManager.staging_dir)
def stage_image(self, index=0):
shutil.copy(TestExifManager.test_images[index],
TestExifManager.staged_images[index])
return TestExifManager.staged_images[index]
def test_init(self):
self.assertIsInstance(ExifManager(), ExifManager)
def test_add_tag(self):
img = self.stage_image(0)
xman = ExifManager()
xman.add_metadata([img], TestExifManager.img_desc)
md = self.get_metadata(img)
self.assertIn('EXIF:ImageDescription', md)
self.assertIn('the image description', md['EXIF:ImageDescription'])
def test_add_nl_tag(self):
img = self.stage_image(0)
xman = ExifManager()
xman.add_metadata([img], TestExifManager.img_desc_nl)
md = self.get_metadata(img)
self.assertIn('EXIF:ImageDescription', md)
self.assertIn(TestExifManager.img_desc_nl.values()[0], md['EXIF:ImageDescription'])
def test_add_xmp_metadata(self):
self.stage_images()
xman = ExifManager()
xman.add_metadata(TestExifManager.staged_images, TestExifManager.xmp_marked)
for img in TestExifManager.staged_images:
md = self.get_metadata(img)
self.assertIn('XMP:Marked', md)
self.assertTrue(os.path.exists(img + "_original"))
def test_add_json_metadata(self):
self.stage_images()
xman = ExifManager()
xman.add_json_metadata(TestExifManager.staged_images, TestExifManager.tag_dict)
for img in TestExifManager.staged_images:
md = self.get_metadata(img)
self.assertIn('XMP:Marked', md)
self.assertIn('XMP:Rights', md)
self.assertTrue(os.path.exists(img + "_original"))
def test_add_xmp_metadata_overwrite(self):
self.stage_images()
xman = ExifManager()
xman.add_metadata(TestExifManager.staged_images, TestExifManager.xmp_marked,
overwrite_original=True)
for img in TestExifManager.staged_images:
md = self.get_metadata(img)
self.assertIn('XMP:Marked', md)
self.assertFalse(os.path.exists(img + "_original"))
def test_add_json_metadata_overwrite(self):
self.stage_images()
xman = ExifManager()
xman.add_json_metadata(TestExifManager.staged_images, TestExifManager.tag_dict,
overwrite_original=True)
for img in TestExifManager.staged_images:
md = self.get_metadata(img)
self.assertIn('XMP:Marked', md)
self.assertIn('XMP:Rights', md)
self.assertFalse(os.path.exists(img + "_original"))
class ExifManager(object):
xmp_re = re.compile(':')
newline_re = re.compile('\n')
logger = logging.getLogger(__name__)
def __init__(self):
self._exiftool = ExifTool()
self._tempfiles = set()
def __del__(self):
self.stop()
def serialize_xmp(self, file_list, **kwargs):
args = {'keep_open': True}
args.update(kwargs)
counter = CountLogger(self.logger, file_list)
counter.count(msg='Serialize XMP', inc=False)
for path in file_list:
self.serialize_xmp_one_file(path, **args)
counter.count(msg="XMP for %s" % (os.path.basename(path), ))
self.stop()
def serialize_xmp_one_file(self, path, **kwargs):
keep_open = kwargs.get('keep_open', False)
kwargs.pop('keep_open', None)
if not self._exiftool.running:
self.start()
xmp = "%s.xmp" % (path, )
tags = ['-tagsFromFile', path, xmp]
self._exiftool.execute(*tags)
if not keep_open:
self.stop()
def get_tag(self, tag, file):
self.start()
value = self._exiftool.get_tag(tag, file)
self.stop()
return value
def add_metadata(self, file_list, prop_dict, overwrite_original=False):
self.start()
for file in file_list:
self._add_md_to_file(file, prop_dict, overwrite_original)
self.stop()
def add_json_metadata(self,
file_list,
prop_dict,
overwrite_original=False):
self.start()
for file in file_list:
self._add_json_md_to_file(file, prop_dict, overwrite_original)
self.stop()
def add_json_one_file(self, filename, prop_dict, **kwargs):
"""
kwargs can be:
overwrite_original : True|[False]
keep_open : True|[False] (keep exiftool instance running)
"""
keep_open = kwargs.get('keep_open', False)
kwargs.pop('keep_open', None)
if not self._exiftool.running:
self.start()
self._add_json_md_to_file(filename, prop_dict, **kwargs)
if not keep_open:
self.stop()
def add_md_one_file(self, filename, prop_dict, **kwargs):
"""
kwargs can be:
overwrite_original : True|[False]
keep_open : True|[False] (keep exiftool instance running)
"""
keep_open = kwargs.get('keep_open', False)
kwargs.pop('keep_open', None)
if not self._exiftool.running:
self.start()
self._add_md_to_file(filename, prop_dict, **kwargs)
if not keep_open:
self.stop()
def _add_json_md_to_file(self, file, prop_dict, overwrite_original=False):
dct = {'SourceFile': file}
dct.update(**prop_dict)
path = self._to_json_file(dct)
tags = ['-json=%s' % path, file]
if overwrite_original:
tags.insert(0, '-overwrite_original')
return self._exiftool.execute(*tags)
def _add_md_to_file(self, file, prop_dict, overwrite_original=False):
tags = self._build_tags(prop_dict)
if overwrite_original:
tags.insert(0, '-overwrite_original')
tags.append(file)
self._exiftool.execute(*tags)
def start(self):
self._exiftool.start()
def stop(self):
self._exiftool.terminate()
self._cleanup()
def _build_tags(self, prop_dict):
tags = []
for key in prop_dict:
value = prop_dict.get(key)
if isinstance(value, list) or isinstance(value, tuple):
for v in value:
tags.append(self._build_tag(key, v))
else:
tags.append(self._build_tag(key, value))
return tags
def _build_tag(self, name, value):
if ExifManager.newline_re.search(value):
path = self._value_to_file(value)
return "%s<=%s" % (self._tag(name), path)
else:
return "%s=%s" % (self._tag(name), value.encode('utf-8'))
def _tag(self, name):
if ExifManager.xmp_re.search(name):
return "-xmp-%s" % name
else:
return "-%s" % name
def _to_json_file(self, prop_dict):
return self._value_to_file(json.dumps(prop_dict))
def _value_to_file(self, value):
f = tempfile.NamedTemporaryFile(delete=False)
f.write(value.encode('utf-8'))
f.flush()
f.seek(0)
name = f.name
self._tempfiles.add(name)
return name
def _cleanup(self):
for f in self._tempfiles:
try:
os.remove(f)
except OSError:
pass
self._tempfiles.clear()
def edit_metadata(original_files_path: str, compressed_files_path: str):
# Rebuild paths
original_files_path = rebuild_dirpath( original_files_path )
compressed_files_path = rebuild_dirpath( compressed_files_path )
original_files = dict()
# Run through original files
with os.scandir( original_files_path ) as file_list:
for file in file_list:
filename_full = file.name
filepath = file.path
filename, extension = os.path.splitext( filename_full )
# Skip iteration if wrong extension
if extension.lower() [1:] not in ORIGINAL_FILE_EXTENSIONS:
continue
# Write to dictionary (overwrite any previous filename)
original_files[filename] = filepath
# Run through compressed files
with os.scandir( compressed_files_path ) as file_list:
for file in file_list:
filename_full = file.name
filepath = file.path
filename, extension = os.path.splitext( filename_full )
# Skip iteration if wrong extension
if extension.lower() [1:] not in COMPRESSED_FILE_EXTENSIONS:
continue
# Create original filename
filename_original = filename.split( COMPRESSED_MARKER ) [0]
# Retrieve original file extension
try:
filepath_original = original_files [filename_original]
except FileNotFoundError:
print(f'{filename_full} - {ERROR_ORIGINAL_NOT_FOUND}')
continue
# Retrieve original file's creation date
with ExifTool() as e:
metadata = e.get_metadata( filepath_original )
modify_date = metadata [0] ['File:FileModifyDate']
# Here can retrieve more metadata to later write using subprocess
# Edit compressed file's EXIF creation date
process = subprocess.Popen(
[
EXIFTOOL_PATH,
f'-AllDates={modify_date}',
'-overwrite_original',
filepath
],
stdin=subprocess.PIPE,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE
)
stdout, stderr = process.communicate()
# Change file's modification time
date_time_obj = datetime.datetime.strptime(modify_date, '%Y:%m:%d %H:%M:%S%z')
mod_time = time.mktime( date_time_obj.timetuple() )
os.utime(filepath, (mod_time, mod_time))
return True
