def extract_xmp(file_path, open_forupdate=False):
"""
This method use xmp lib to extra xmp metadata
:param file_path: path
:param open_forupdate: False by default
:return: unicode string representation of xmp metadata
"""
xmpfile = libxmp.XMPFiles(file_path=file_path,
open_forupdate=open_forupdate)
libxmp.XMPFiles()
xmp = xmpfile.get_xmp()
xmpfile.close_file()
return xmp.serialize_to_unicode()
def apply_keyword_local(pair):
if 'Full Collection' in pair.album:
pair.keyword = pair.album.lstrip('Full Collection')
elif 'Album' in pair.album:
pair.keyword = pair.album.lstrip('Album')
else:
pair.keyword = pair.album
try:
xmpfile = libxmp.XMPFiles(file_path=os.path.join(
pair.local_path, pair.local_fn),
open_forupdate=True)
xmp = xmpfile.get_xmp()
#Using the dict, you can get a full dump of all file contens:
# xmpdict = libxmp.utils.object_to_dict(xmp)
current_keywords = xmp.get_property(libxmp.consts.XMP_NS_DC,
'subject[1]')
if not pair.keyword in current_keywords: #See: http://purl.org/dc/elements/1.1/ in the dict or online
if not current_keywords[-1] == ',' and len(current_keywords) > 0:
pair.keyword = ',' + pair.keyword
xmp.set_property(libxmp.consts.XMP_NS_DC, 'subject[1]',
current_keywords + pair.keyword)
if xmpfile.can_put_xmp(xmp):
xmpfile.put_xmp(xmp)
xmpfile.close_file()
except:
pass
return pair
def process(fullpath, config, rcontext, columns=None):
# Try to parse the GIF data
try:
image = Image.open(fullpath, "r")
assorted = [
image.info['version'],
image.info['duration'],
image.info['transparency'],
image.info['background'],
]
# Seek until EOF to find the number of animation frames
noframes = 0
try:
while True:
noframes += 1
image.seek(image.tell() + 1)
except EOFError:
pass
assorted.append(noframes)
if 'loop' in image.info:
assorted.append(image.info['loop'])
else:
assorted.append(0)
if 'extension' in image.info:
assorted.append(image.info['extension'])
else:
assorted.append(None)
# Close the PIL image handler
del image
# Store the embedded XMP metadata
if config.ENABLEXMP:
import libxmp
xmpfile = libxmp.XMPFiles(file_path=fullpath)
assorted.append(str(xmpfile.get_xmp()))
xmpfile.close_file()
else:
assorted.append(None)
# Make sure we stored exactly the same amount of columns as
# specified!!
assert len(assorted) == len(columns)
# Print some data that is stored in the database if debug is true
if config.DEBUG:
print "\nGIF file data:"
for i in range(0, len(assorted)):
print "%-18s %s" % (columns[i], assorted[i])
print
return assorted
except:
traceback.print_exc(file=sys.stderr)
return None
def process(fullpath, config, rcontext, columns=None):
# Try to parse TGA data
try:
image = Image.open(fullpath, "r")
assorted = [image.tile]
info_dictionary = image.info
# Check if compression is in info dictionary,
# if so put it in our list
if "compression" in info_dictionary:
assorted.append(info_dictionary["compression"])
info_dictionary.pop("compression")
else:
assorted.append(None)
# Check if orientation is in info dictionary,
# if so put it in our list
if "orientation" in info_dictionary:
assorted.append(info_dictionary["orientation"])
info_dictionary.pop("orientation")
else:
assorted.append(None)
# If there are still other values in the
# dict then put those in column
assorted.append(info_dictionary)
# Delete variable
del image, info_dictionary
# Store the embedded XMP metadata
if config.ENABLEXMP:
import libxmp
xmpfile = libxmp.XMPFiles(file_path=fullpath)
assorted.append(str(xmpfile.get_xmp()))
xmpfile.close_file()
else:
assorted.append(None)
# Make sure we stored exactly the same amount of columns as
# specified!!
assert len(assorted) == len(columns)
# Print some data that is stored in the database if debug is true
if config.DEBUG:
print "\nTGA file data:"
for i in range(0, len(assorted)):
print "%-18s %s" % (columns[i], assorted[i])
print
return assorted
except:
traceback.print_exc(file=sys.stderr)
# Store values in database so not the whole application crashes
return None
def get_dc_from_file(filename):
'''Get an embedded XMP from a media file (PDF, JPEG, …) and extract the
Dublin Core elements.
Keyword arguments:
filename -- name of the media file
'''
xmpfile = libxmp.XMPFiles(file_path=filename)
xmp = xmpfile.get_xmp()
return {} if type(xmp) is NoneType else __get_dc(xmp)
def sync_iptc(self, file_path: Path) -> None:
"""
Sync metadata to file.
Take in a file path because there's no guarantee file is at
self.get_file_save_path()
Note to future self: it doesn't matter if you put a namespace in the
name like "xmp:CreateDate" vs "CreateDate" or "dc:creator" vs "creator"
"""
if self.file_ext not in ("jpg", "jpeg", "png"):
return
xmpfile = libxmp.XMPFiles(file_path=str(file_path),
open_forupdate=True)
xmp = xmpfile.get_xmp()
if not xmp:
# TODO why do some PNG files not return xmp?
return
# Existing meta, this isn't very useful on it's own
# xmpdict = libxmp.utils.object_to_dict(xmp)
# Set simple metadata fields
# https://www.iptc.org/std/photometadata/specification/IPTC-PhotoMetadata#date-created
try:
xmp.get_property(XMP_NS_XMP, "CreateDate")
except libxmp.XMPError:
xmp.set_property(XMP_NS_XMP, "CreateDate", self.created_at)
try:
xmp.get_property(XMP_NS_XMP, "ModifyDate")
except libxmp.XMPError:
xmp.set_property(XMP_NS_XMP, "ModifyDate", self.updated_at)
try:
xmp.get_property(XMP_NS_XMP, "Rating")
except libxmp.XMPError:
# If we bookmarked it, we must like it, so set a default of "4"
xmp.set_property(XMP_NS_XMP, "Rating", "4")
# https://www.iptc.org/std/photometadata/specification/IPTC-PhotoMetadata#creator
add_array_xmp(xmp, "creator", self.artists)
# https://www.iptc.org/std/photometadata/specification/IPTC-PhotoMetadata#keywords
add_array_xmp(
xmp,
"subject",
self.copyright + self.characters +
self.tag_string_general.split(" "),
)
if xmpfile.can_put_xmp(xmp):
xmpfile.put_xmp(xmp)
xmpfile.close_file()
def read_xmp_metadata(self, file):
# Lecture du fichier
xmpfile = libxmp.XMPFiles(file_path=file, open_forupdate=False)
# Récupérer le XML
xmp = str(xmpfile.get_xmp())
xmp = "<rdf:RDF" + xmp.split("<rdf:RDF", 1)[1]
xmp = xmp.split("</rdf:RDF>", 1)[0] + "</rdf:RDF>"
# Fermer le fichier
xmpfile.close_file()
# Retourner le contenu RDF
return xmp
def embed_xmp_bytes(image: io.BytesIO, work_properties):
"""
Given a file-like `io.BytesIO` object, embed ccREL metadata inside of it.
For our purposes, we assume that the file is an image.
:param image: A BytesIO representation of an image.
:param work_properties: A dictionary with keys 'license_url' and
'attribution'. 'creator', and 'work_landing_page' are optional (but highly
recommended)
:return: An `io.BytesIO` object containing XMP metadata.
"""
# libxmp only works with actual file locations on the disk. To work around
# this limitation, rather than embedding the metadata directly into the
# `io.BytesIO` object, we have to use a temporary file and then convert it
# back.
# https://github.com/python-xmp-toolkit/python-xmp-toolkit/issues/46
filename = '/tmp/{}'.format(uuid.uuid4())
with open(filename, 'w+b') as xmp_temp:
xmp_temp.write(image.getvalue())
xmp_temp.flush()
xmpfile = libxmp.XMPFiles(file_path=xmp_temp.name, open_forupdate=True)
# Set CC rights.
xmp = xmpfile.get_xmp()
xmp.register_namespace(XMP_NS_CC, 'cc')
xmp.set_property(XMP_NS_CC, 'license', work_properties['license_url'])
if 'creator' in work_properties:
if not xmp.does_property_exist(XMP_NS_CC, 'attributionName'):
xmp.set_property(XMP_NS_CC, 'attributionName',
work_properties['creator'])
if 'work_landing_page' in work_properties:
if not xmp.does_property_exist(XMP_NS_CC, 'attributionURL'):
xmp.set_property(XMP_NS_CC, 'attributionURL',
work_properties['work_landing_page'])
xmp.register_namespace(XMP_NS_XMP, 'xmp')
if 'identifier' in work_properties:
if not xmp.does_property_exist(XMP_NS_XMP, 'Identifier'):
xmp.set_property(XMP_NS_XMP, 'Identifier',
work_properties['identifier'])
# Set generic XMP rights.
xmp.register_namespace(XMP_NS_XMP_Rights, 'xmpRights')
if not xmp.does_property_exist(XMP_NS_XMP_Rights, 'XMP_NS_XMP_Rights'):
xmp.set_property_bool(XMP_NS_XMP_Rights, 'Marked', True)
if not xmp.does_property_exist(XMP_NS_XMP_Rights, 'UsageTerms'):
usage = work_properties['attribution']
xmp.set_property(XMP_NS_XMP_Rights, 'UsageTerms', usage)
xmpfile.put_xmp(xmp)
xmpfile.close_file()
with open(filename, 'r+b') as xmpfile:
file_with_xmp = io.BytesIO(xmpfile.read())
os.remove(filename)
return file_with_xmp
def put_xmp_to_file(xmp_meta, file_path):
xmp_file = libxmp.XMPFiles(file_path=file_path, open_forupdate=True)
if not xmp_file.can_put_xmp(xmp_meta):
if not file_format_supported(file_path):
raise Exception("Image format of %s does not allow metadata" %
(file_path))
else:
raise Exception("Could not add metadata to image %s" % (file_path))
xmp_file.put_xmp(xmp_meta)
xmp_file.close_file()
def wav_to_mp3(wavfn, delete_wav=False, lame_quality="V 2"):
mp3fn = ".".join(wavfn.split('.')[:-1]) + '.mp3'
check_output('lame -{} "{}"'.format(lame_quality, wavfn), shell=True)
if LIBXMP:
xmpfile = libxmp.XMPFiles(file_path=wavfn)
xmpfile2 = libxmp.XMPFiles(file_path=mp3fn, open_forupdate=True)
xmp = xmpfile.get_xmp()
try:
if xmpfile2.can_put_xmp(xmp):
xmpfile2.put_xmp(xmp)
else:
print "Could not convert xmp from wav to mp3"
except:
print "File has no xmp data"
xmpfile.close_file()
xmpfile2.close_file()
if delete_wav:
check_output('rm "{}"'.format(wavfn), shell=True)
def open(self):
if not self.is_open:
# Open file for updating
self.xmpfile = libxmp.XMPFiles(file_path=self.filename,
open_forupdate=True)
# Try to read existing XMP
self.xmp = self.xmpfile.get_xmp()
# Make new XMP if not exist
if self.xmp is None:
self.xmp = libxmp.core.XMPMeta()
self.is_open = True
# Get tags
temp = self.get_tags()
def process(file, config, rcontext, columns=None):
fullpath = file.fullpath
try:
# open the wave file
wave_file = wave.open(fullpath, "rb")
# fill variables from the wave file, (nchannels, sampwidth,
# framerate, nframes, comptype, compname)
assorted = list(wave_file.getparams())
# duration of the wave file is amount of frames divided by framerate
assorted.append(wave_file.getnframes() /
float(wave_file.getframerate()))
# close the wavefile first before opening using the XMP toolkit
wave_file.close()
# Store the embedded XMP metadata
if config.ENABLEXMP:
import libxmp
xmpfile = libxmp.XMPFiles(file_path=fullpath)
assorted.append(str(xmpfile.get_xmp()))
xmpfile.close_file()
else:
assorted.append(None)
# Make sure we stored exactly the same amount of columns as
# specified!!
assert len(assorted) == len(columns)
# Print some data that is stored in the database if debug is true
if config.DEBUG:
print "\nWAV file data:"
for i in range(0, len(assorted)):
print "%-18s %s" % (columns[i], assorted[i])
print
return assorted
except:
traceback.print_exc(file=sys.stderr)
return None
def metadataToFile(result, fileDict):
for row in result:
# TODO: Check for existing file in fileDict. If not existing, KeyError will be raised as well.
try:
path = fileDict[row['Dateiname']]
print("------------------------------")
print(path)
xmpfile = libxmp.XMPFiles(file_path=path, open_forupdate=True)
xmp = xmpfile.get_xmp()
# Edit each of the xmp attributes of the image
for key, value in mapping.mapping.items():
try:
if row[key] != "":
value_split = value.strip().split(':', 1)
# if option -o: the original xmp data will be deleted
if overrideMD == 1:
xmp.delete_property(
eval(value_split[0]),
unicode(value_split[1], 'utf-8'))
xmp.set_property(eval(value_split[0]),
unicode(value_split[1], 'utf-8'),
unicode(row[key], 'utf-8'))
except libxmp.XMPError:
errorMessage = 'Attribute ' + value.upper(
) + ' already existing for file: ' + row['Dateiname']
print errorMessage
logging.debug(errorMessage)
# Save xmp metadata to the file
xmpfile.put_xmp(xmp)
xmpfile.close_file()
except KeyError:
errorMessage = 'KeyError found, while processing ' + row[
'Dateiname'] + '. Wrong mapping.py settings or missing File?'
print errorMessage
logging.debug(errorMessage)
def process(file, config, rcontext, columns=None):
fullpath = file.fullpath
# Try to parse PNG data
try:
image = Image.open(fullpath, "r")
assorted = [image.tile, image.text]
info_dictionary = image.info
# Check if ICC profile is in info dictionary
# if so put it in our list
if "icc_profile" in info_dictionary:
assorted.append(info_dictionary["icc_profile"])
info_dictionary.pop("icc_profile")
else:
assorted.append(None)
# Check if interlace is in info dictionary
# if so put it in our list
if "interlace" in info_dictionary:
assorted.append(info_dictionary["interlace"])
info_dictionary.pop("interlace")
else:
assorted.append(None)
# Check if transparency is in info dictionary
# if so put it in our list
if "transparency" in info_dictionary:
assorted.append(info_dictionary["transparency"])
info_dictionary.pop("transparency")
else:
assorted.append(None)
# Check if gamma is in info dictionary, if so put it in our list
if "gamma" in info_dictionary:
assorted.append(info_dictionary["gamma"])
info_dictionary.pop("gamma")
else:
assorted.append(None)
# Check if dpi is in info dictionary, if so put it in our list
if "dpi" in info_dictionary:
assorted.append(info_dictionary["dpi"][0])
assorted.append(info_dictionary["dpi"][1])
info_dictionary.pop("dpi")
else:
assorted.append(None)
assorted.append(None)
# Check if aspect is in info dictionary, if so put it in our list
if "aspect" in image.info:
assorted.append(info_dictionary["aspect"])
info_dictionary.pop("aspect")
else:
assorted.append(None)
# If there are still other values in the dict
# then put those in column
assorted.append(info_dictionary)
# Delete variable
del info_dictionary, image
# Store the embedded XMP metadata
if config.ENABLEXMP:
import libxmp
xmpfile = libxmp.XMPFiles(file_path=fullpath)
assorted.append(str(xmpfile.get_xmp()))
xmpfile.close_file()
else:
assorted.append(None)
# Make sure we stored exactly the same amount of columns as
# specified!!
assert len(assorted) == len(columns)
# Print some data that is stored in the database if debug is true
if config.DEBUG:
print "\nPNG file data:"
for i in range(0, len(assorted)):
print "%-18s %s" % (columns[i], assorted[i])
print
return assorted
except:
traceback.print_exc(file=sys.stderr)
# Store values in database so not the whole application crashes
return None
import base64
import io
import libxmp
import logging
import zipfile
parser = argparse.ArgumentParser(
description='Extract files from image metadata')
parser.add_argument('image', help='Image with embedded files in metadata')
def extract_payload(payload):
payload_io = io.BytesIO(base64.b64decode(payload))
with zipfile.ZipFile(
payload_io, mode='r', compression=zipfile.ZIP_LZMA) as zip_file:
for filename in zip_file.namelist():
logging.info(f'Extracting {filename}')
zip_file.extract(filename)
if __name__ == '__main__':
args = parser.parse_args()
logging.basicConfig(level=logging.DEBUG)
xmp_file = libxmp.XMPFiles(file_path=args.image)
xmp_meta = xmp_file.get_xmp()
if xmp_meta is None:
print(f'Image {args.image} does not contain any embedded files')
extract_payload(xmp_meta.get_property(
libxmp.consts.XMP_NS_DC, u'embedded_payload'))
def process(fullpath, config, rcontext, columns=None):
# The whole parse data method is in a try block to catch any exception
try:
# Read the audio file and get the two data
# classes from it (Tag and AudioInfo)
track = eyed3.load(fullpath)
track_tag = track.tag
track_info = track.info
# A variable to store mutiple strings in a list
list_of_strings = []
# Init list
assorted = []
# Store the title in the list
assorted.append(track_tag.title)
'''
To get a tag from the audio file we need to set a frame
in the eyed3 lib. In return we get mutiple frames or NoneType.
For each frame we get the correct variable and this is stored
in a list. After the loop we parse all found data into one string,
this string for example look like: subtitle / subtitle / subtitle
*The eyed3 lib says that it will never occur that there are
multiple frames, that is why we store all data in one string,
but to be sure of this, we still irritate through all frames*
Because a NoneType can not be irritated we use the "or",
so when we get a NoneType the for loop uses an empty list.
Wich will result in an empty string because
nothing is added to the list.
'''
# Store the subtitle in the list
for subtitle_frame in track_tag.frame_set["TIT3"] or []:
list_of_strings.append(subtitle_frame.text)
assorted.append(' / '.join(list_of_strings))
# Store the artist in the list
assorted.append(track_tag.artist)
# Store the album artist in the list
list_of_strings = []
for album_artist_frame in track_tag.frame_set["TPE2"] or []:
list_of_strings.append(album_artist_frame.text)
assorted.append(' / '.join(list_of_strings))
# Store the album in the list
assorted.append(track_tag.album)
'''
Some tags return an array or list of items.
To get the correct data of these lists we first need to know
if the list is not a NoneType or empty.
If we don't check this an empty list will result in an exception,
"[0]" can not be used an such list
If the list is empty we still need to give something in return
to the database, so we first define the variable to None.
'''
# Store the track_number in the list
track_number = None
track_total = None
if (track_tag.track_num):
track_number = track_tag.track_num[0]
track_total = track_tag.track_num[1]
assorted.append(track_number)
assorted.append(track_total)
# Store the disc_number in the list
disc_number = None
disc_total = None
if (track_tag.disc_num):
disc_number = track_tag.disc_num[0]
disc_total = track_tag.disc_num[1]
assorted.append(disc_number)
assorted.append(disc_total)
# delete variables
del track_number, track_total, disc_number, disc_total
# Store the cd_id in the list
assorted.append(track_tag.cd_id)
# Store the publisher in the list
assorted.append(track_tag.publisher)
# Store the composer in the list
list_of_strings = []
for composer_frame in track_tag.frame_set["TCOM"] or []:
list_of_strings.append(composer_frame.text)
assorted.append(' / '.join(list_of_strings))
# Store the conductor in the list
list_of_strings = []
for conductor_frame in track_tag.frame_set["TPE3"] or []:
list_of_strings.append(conductor_frame.text)
assorted.append(' / '.join(list_of_strings))
# Store the group content in the list
list_of_strings = []
for group_content_frame in track_tag.frame_set["TIT1"] or []:
list_of_strings.append(group_content_frame.text)
assorted.append(' / '.join(list_of_strings))
# Store the releasedate and the recording year in the list
assorted.append(track_tag.release_date)
assorted.append(track_tag.recording_date)
# Store beats per minute in the list
assorted.append(track_tag.bpm)
# Store the duration of the song in the list
assorted.append(track_info.time_secs)
# Store the play count of the song in the list
assorted.append(track_tag.play_count)
# Store the terms of use in the list
assorted.append(track_tag.terms_of_use)
# Store the language in the list
list_of_strings = []
for language_frame in track_tag.frame_set["TLAN"] or []:
list_of_strings.append(language_frame.text)
assorted.append(' / '.join(list_of_strings))
# Store the rating in the list
rating = 0
for rating in track_tag.popularities:
rating = rating.rating
assorted.append(rating)
# Store the genre in the list
genre = None
if (track_tag.genre):
genre = track_tag.genre.name
assorted.append(genre)
# Store the comment data in the list
comment_description = None
comment_lang = None
comment_text = None
for comment in track_tag.comments:
comment_description = comment.description
comment_lang = comment.lang
comment_text = comment.text
assorted.append(comment_text)
assorted.append(comment_description)
assorted.append(comment_lang)
# delete variables
del rating, genre, comment_description, comment_lang, comment_text
# Store the encoded by in the list
list_of_strings = []
for encoded_by_frame in track_tag.frame_set["TENC"] or []:
list_of_strings.append(encoded_by_frame.text)
assorted.append(' / '.join(list_of_strings))
# Store the copyright in the list
list_of_strings = []
for copyright_frame in track_tag.frame_set["TCOP"] or []:
list_of_strings.append(copyright_frame.text)
assorted.append(' / '.join(list_of_strings))
# Store the mood in the list
list_of_strings = []
for mood_frame in track_tag.frame_set["TMOO"] or []:
list_of_strings.append(mood_frame.text)
assorted.append(' / '.join(list_of_strings))
# Store the compilation in the list
list_of_strings = []
for compitation_frame in track_tag.frame_set["TPE4"] or []:
list_of_strings.append(compitation_frame.text)
assorted.append(' / '.join(list_of_strings))
# Store the user text data in the list
user_text_description = None
user_text_text = None
for user_text in track_tag._user_texts:
user_text_description = user_text.description
user_text_text = user_text.text
assorted.append(user_text_text)
assorted.append(user_text_description)
# Store the lyrics data in the list
lyrics_description = None
lyrics_text = None
lyrics_lang = None
for lyric in track_tag.lyrics:
lyrics_description = lyric.description
lyrics_text = lyric.text
lyrics_lang = lyric.lang
assorted.append(lyrics_text)
assorted.append(lyrics_description)
assorted.append(lyrics_lang)
# Store the image data in the list
image_description = None
image_url = None
image_picturetype = None
for image in track_tag.images:
image_description = image.description
image_url = image.image_url
image_picturetype = image.picTypeToString(image.picture_type)
assorted.append(image_description)
assorted.append(image_picturetype)
assorted.append(image_url)
# delete variables
del user_text_description, user_text_text, lyrics_description
del lyrics_lang, lyrics_text, image_description, image_url
del image_picturetype
# Store the chapter data in the list
chapter_title = None
chapter_subtitle = None
chapter_starttime = None
chapter_endtime = None
chapter_startoffset = None
chapter_endoffset = None
for chapter in track_tag.chapters:
chapter_title = chapter.title
chapter_subtitle = chapter.subtitle
if (chapter.times):
chapter_starttime = chapter.times[0]
chapter_endtime = chapter.times[1]
if (chapter.offsets):
chapter_startoffset = chapter.offsets[0]
chapter_endoffset = chapter.offsets[1]
assorted.append(chapter_title)
assorted.append(chapter_subtitle)
assorted.append(chapter_starttime)
assorted.append(chapter_endtime)
assorted.append(chapter_startoffset)
assorted.append(chapter_endoffset)
# delete variables
del chapter_title, chapter_subtitle, chapter_starttime
del chapter_endtime, chapter_startoffset, chapter_endoffset
# Store all URL's in the list
assorted.append(track_tag.commercial_url)
assorted.append(track_tag.copyright_url)
assorted.append(track_tag.artist_url)
assorted.append(track_tag.audio_file_url)
assorted.append(track_tag.audio_source_url)
assorted.append(track_tag.internet_radio_url)
assorted.append(track_tag.payment_url)
assorted.append(track_tag.publisher_url)
user_url = None
for user_url_tag in track_tag._user_urls:
user_url = user_url_tag.url
assorted.append(user_url)
# Delete all variables that are not used anymore
del user_url, list_of_strings, track_info, track_tag, track
# Get APEv2 tag from MP3,
# because we don't know the key:value we store all data in one column
try:
apev2_tag = APEv2(fullpath)
except:
apev2_tag = None
assorted.append(apev2_tag)
# Delete variable
del apev2_tag
# Store the embedded XMP metadata
if config.ENABLEXMP:
import libxmp
xmpfile = libxmp.XMPFiles(file_path=fullpath)
assorted.append(str(xmpfile.get_xmp()))
xmpfile.close_file()
else:
assorted.append(None)
# Make sure we stored exactly the same amount of columns as
# specified!!
assert len(assorted) == len(columns)
# Print some data that is stored in the database if debug is true
if config.DEBUG:
print "\nMPEG file data:"
for i in range(0, len(assorted)):
print "%-18s %s" % (columns[i], assorted[i])
print
# Store in database
return assorted
except:
traceback.print_exc(file=sys.stderr)
return None
def export(self, **kwargs):
"""
Generate audio file from composition.
:param str. filename: Output filename (no extension)
:param str. filetype: Output file type (only .wav supported for now)
:param integer samplerate: Sample rate of output audio
:param integer channels: Channels in output audio, if different than originally specified
:param bool. separate_tracks: Also generate audio file for each track in composition
:param int min_length: Minimum length of output array (in frames). Will zero pad extra length.
:param bool. adjust_dynamics: Automatically adjust dynamics (will document later)
"""
# get optional args
filename = kwargs.pop('filename', 'out')
filetype = kwargs.pop('filetype', 'wav')
adjust_dynamics = kwargs.pop('adjust_dynamics', False)
samplerate = kwargs.pop('samplerate', None)
channels = kwargs.pop('channels', self.channels)
separate_tracks = kwargs.pop('separate_tracks', False)
min_length = kwargs.pop('min_length', None)
if samplerate is None:
samplerate = np.min([track.samplerate for track in self.tracks])
encoding = 'pcm16'
to_mp3 = False
if filetype == 'ogg':
encoding = 'vorbis'
elif filetype == 'mp3':
filetype = 'wav'
to_mp3 = True
if separate_tracks:
# build the separate parts of the composition if desired
for track in self.tracks:
out = self.build(track_list=[track],
adjust_dynamics=adjust_dynamics,
min_length=min_length,
channels=channels)
out_file = Sndfile(
"%s-%s.%s" % (filename, track.name, filetype), 'w',
Format(filetype, encoding=encoding), channels, samplerate)
out_file.write_frames(out)
out_file.close()
# always build the complete composition
out = self.build(adjust_dynamics=adjust_dynamics,
min_length=min_length,
channels=channels)
out_filename = "%s.%s" % (filename, filetype)
out_file = Sndfile(out_filename, 'w',
Format(filetype, encoding=encoding), channels,
samplerate)
out_file.write_frames(out)
out_file.close()
if LIBXMP and filetype == "wav":
xmp = libxmp.XMPMeta()
ns = libxmp.consts.XMP_NS_DM
p = xmp.get_prefix_for_namespace(ns)
xpath = p + 'Tracks'
xmp.append_array_item(ns,
xpath,
None,
array_options={"prop_value_is_array": True},
prop_value_is_struct=True)
xpath += '[1]/' + p
xmp.set_property(ns, xpath + "trackName", "CuePoint Markers")
xmp.set_property(ns, xpath + "trackType", "Cue")
xmp.set_property(ns, xpath + "frameRate", "f%d" % samplerate)
for i, lab in enumerate(self.labels):
xmp.append_array_item(
ns,
xpath + "markers",
None,
array_options={"prop_value_is_array": True},
prop_value_is_struct=True)
xmp.set_property(ns, xpath + "markers[%d]/%sname" % (i + 1, p),
lab.name)
xmp.set_property(
ns, xpath + "markers[%d]/%sstartTime" % (i + 1, p),
str(lab.sample(samplerate)))
xmpfile = libxmp.XMPFiles(file_path=out_filename,
open_forupdate=True)
if xmpfile.can_put_xmp(xmp):
xmpfile.put_xmp(xmp)
xmpfile.close_file()
if to_mp3:
wav_to_mp3(out_filename, delete_wav=True)
return out
# Returns base64-encoded ZIP archive with all files in file listing
def generate_payload(file_listing):
zip_buffer = io.BytesIO()
with zipfile.ZipFile(zip_buffer, mode='w',
compression=zipfile.ZIP_LZMA) as zip_file:
for filename in file_listing:
logging.info(f'Compressing file {filename}')
zip_file.write(filename, os.path.basename(filename))
return base64.b64encode(zip_buffer.getbuffer()).decode('utf8')
if __name__ == '__main__':
args = parser.parse_args()
logging.basicConfig(level=logging.DEBUG)
# Create a copy of the input file
with open(args.output, 'wb') as output_io:
output_io.write(open(args.input, 'rb').read())
logging.info(f'Reading input image {args.input}')
xmp_file = libxmp.XMPFiles(file_path=args.output, open_forupdate=True)
xmp_meta = libxmp.XMPMeta()
xmp_meta.set_property(libxmp.consts.XMP_NS_DC, u'embedded_payload',
generate_payload(args.filename))
logging.info(f'Writing output image {args.output}')
xmp_file.put_xmp(xmp_meta)
xmp_file.close_file()
def main():
# Program options management
parser = OptionParser(
usage="%prog imageFile [options]",
version="0.1",
description=
"This tool generates multi-resolution astronomical images from large image for display by Stellarium/VirGO. The passed image is split in small tiles and the corresponding index JSON files are generated. Everything is output in the current directory.",
formatter=IndentedHelpFormatter(max_help_position=33, width=80))
parser.add_option("-f",
"--fitsheader",
dest="fitsHeader",
help="use the FITS header from FILE to get WCS info",
metavar="FILE")
parser.add_option("-g",
"--gzipcompress",
dest="gzipCompress",
action="store_true",
default=False,
help="compress the produced JSON index using gzip")
parser.add_option(
"-t",
"--tilesize",
dest="tileSize",
type="int",
default=256,
help="output image tiles size in pixel (default: %default)",
metavar="SIZE")
parser.add_option("-i",
"--onlyindex",
dest="makeImageTiles",
action="store_false",
default=True,
help="output only the JSON index")
parser.add_option(
"-l",
"--maxLevPerIndex",
dest="maxLevelPerIndex",
default=3,
type="int",
help="put up to MAX levels per index file (default: %default)",
metavar="MAX")
parser.add_option(
"-b",
"--maxBrightness",
dest="maxBrightness",
default=13.,
type="float",
help=
"the surface brightness of a white pixel of the image in V mag/arcmin^2 (default: %default)",
metavar="MAG")
parser.add_option("-a",
"--alphaBlend",
dest="alphaBlend",
action="store_true",
default=False,
help="activate alpha blending for this image")
parser.add_option("--imgInfoShort",
dest="imgInfoShort",
type="string",
help="the short name of the image",
metavar="STR")
parser.add_option("--imgInfoFull",
dest="imgInfoFull",
type="string",
help="the full name of the image",
metavar="STR")
parser.add_option("--imgInfoUrl",
dest="imgInfoUrl",
type="string",
help="the info URL about the image",
metavar="STR")
parser.add_option("--imgCreditsShort",
dest="imgCreditsShort",
type="string",
help="the short name of the image creator",
metavar="STR")
parser.add_option("--imgCreditsFull",
dest="imgCreditsFull",
type="string",
help="the full name of the image creator",
metavar="STR")
parser.add_option("--imgCreditsInfoUrl",
dest="imgCreditsInfoUrl",
type="string",
help="the info URL about the image creator",
metavar="STR")
parser.add_option("--srvCreditsShort",
dest="serverCreditsShort",
type="string",
help="the short name of the hosting server",
metavar="STR")
parser.add_option("--srvCreditsFull",
dest="serverCreditsFull",
type="string",
help="the full name of the hosting server",
metavar="STR")
parser.add_option("--srvCreditsInfoUrl",
dest="serverCreditsInfoUrl",
type="string",
help="the info URL about the hosting server",
metavar="STR")
(options, args) = parser.parse_args()
# Need at least an image file as input
if len(args) < 1:
print "Usage: " + os.path.basename(
sys.argv[0]) + " imageFile [options]"
exit(0)
imgFile = sys.argv[1]
# We now have valid arguments
if options.fitsHeader != None:
# Try to read the provided FITS header file to extract the WCS
wcs = astWCS.WCS(options.fitsHeader)
else:
# Else try to generate the WCS from the xmp informations contained in the file header
import libxmp
print "Try to import WCS info from the XMP headers in the image"
libxmp.XMPFiles.initialize()
xmpfile = libxmp.XMPFiles()
xmpfile.open_file(imgFile, libxmp.consts.XMP_OPEN_READ)
xmp = xmpfile.get_xmp()
ns = 'http://www.communicatingastronomy.org/avm/1.0/'
nxpix = int(
xmp.get_array_item(ns, 'Spatial.ReferenceDimension', 1).keys()[0])
nypix = int(
xmp.get_array_item(ns, 'Spatial.ReferenceDimension', 2).keys()[0])
ctype1 = 'RA---TAN'
ctype2 = 'DEC--TAN'
crpix1 = float(
xmp.get_array_item(ns, 'Spatial.ReferencePixel', 1).keys()[0])
crpix2 = float(
xmp.get_array_item(ns, 'Spatial.ReferencePixel', 2).keys()[0])
crval1 = float(
xmp.get_array_item(ns, 'Spatial.ReferenceValue', 1).keys()[0])
crval2 = float(
xmp.get_array_item(ns, 'Spatial.ReferenceValue', 2).keys()[0])
cdelt1 = float(xmp.get_array_item(ns, 'Spatial.Scale', 1).keys()[0])
cdelt2 = float(xmp.get_array_item(ns, 'Spatial.Scale', 2).keys()[0])
crota = float(xmp.get_property(ns, "Spatial.Rotation"))
equinox = xmp.get_property(ns, "Spatial.Equinox")
if equinox == 'J2000':
equinox = 2000
elif equinox == 'B1950':
equinox = 1950
else:
equinox = float(equinox)
coordFrameCstr = xmp.get_property(ns, "Spatial.CoordinateFrame")
header = astWCS.pyfits.Header()
header.update('SIMPLE', 'T')
header.update('BITPIX', 8)
header.update('NAXIS', 2)
header.update('NAXIS1', nxpix)
header.update('NAXIS2', nypix)
header.update('CDELT1', cdelt1)
header.update('CDELT2', cdelt2)
header.update('CTYPE1', 'RA---TAN')
header.update('CTYPE2', 'DEC--TAN')
header.update('CRVAL1', crval1)
header.update('CRVAL2', crval2)
header.update('CRPIX1', crpix1)
header.update('CRPIX2', crpix2)
header.update('CROTA', crota)
header.update('EQUINOX', equinox)
header.update('RADECSYS', coordFrameCstr)
wcs = astWCS.WCS(header, mode='pyfits')
im = Image.open(imgFile)
nbTileX = (im.size[0] + options.tileSize) // options.tileSize
nbTileY = (im.size[1] + options.tileSize) // options.tileSize
maxSize = max(im.size[0], im.size[1])
nbLevels = 0
while 2**nbLevels * options.tileSize < maxSize:
nbLevels += 1
print "Will tesselate image (", im.size[0], "x", im.size[
1], ") in", nbTileX, 'x', nbTileY, 'tiles on', nbLevels + 1, 'levels'
# Create the master level 0 tile, which recursively creates the subtiles
masterTile = createTile(0, nbLevels, 0, 0, wcs, im, options.makeImageTiles,
options.tileSize)
# Add some top-level informations
masterTile.maxBrightness = options.maxBrightness
masterTile.alphaBlend = options.alphaBlend
masterTile.imageInfo.short = options.imgInfoShort
masterTile.imageInfo.full = options.imgInfoFull
masterTile.imageInfo.infoUrl = options.imgInfoUrl
masterTile.imageCredits.short = options.imgCreditsShort
masterTile.imageCredits.full = options.imgCreditsFull
masterTile.imageCredits.infoUrl = options.imgCreditsInfoUrl
masterTile.serverCredits.short = options.serverCreditsShort
masterTile.serverCredits.full = options.serverCreditsFull
masterTile.serverCredits.infoUrl = options.serverCreditsInfoUrl
# masterTile contains the whole tree, just need to output it as desired
masterTile.outputJSON(qCompress=options.gzipCompress,
maxLevelPerFile=options.maxLevelPerIndex)
def get_xmp_from_file(file_path):
return libxmp.XMPFiles(file_path=file_path).get_xmp()
def process(fullpath, config, rcontext, columns=None):
# Try to parse TIFF data
try:
image = Image.open(fullpath, "r")
assorted = [image.tile]
info_dictionary = image.info
# Check if ICC profile is in info dictionary,
# if so put it in our list
if "icc_profile" in info_dictionary:
assorted.append(info_dictionary["icc_profile"])
info_dictionary.pop("icc_profile")
else:
assorted.append(None)
# Check if compression is in info dictionary,
# if so put it in our list
if "compression" in info_dictionary:
assorted.append(info_dictionary["compression"])
info_dictionary.pop("compression")
else:
assorted.append(None)
# Check if dpi is in info dictionary, if so put it in our list
if "dpi" in info_dictionary:
assorted.append(info_dictionary["dpi"][0])
assorted.append(info_dictionary["dpi"][1])
info_dictionary.pop("dpi")
else:
assorted.append(None)
assorted.append(None)
# Check if resolution is in info dictionary,
# if so put it in our list
if "resolution" in info_dictionary:
assorted.append(info_dictionary["resolution"][0])
assorted.append(info_dictionary["resolution"][1])
info_dictionary.pop("resolution")
else:
assorted.append(None)
assorted.append(None)
# If there are still other values in the
# dict then put those in column
assorted.append(info_dictionary)
# Get all values from tag atribute, EXIF Tags
assorted.append(image.tag.get(TiffImagePlugin.BITSPERSAMPLE))
assorted.append(
image.tag.get(TiffImagePlugin.PHOTOMETRIC_INTERPRETATION))
assorted.append(image.tag.get(TiffImagePlugin.FILLORDER))
assorted.append(image.tag.get(TiffImagePlugin.IMAGEDESCRIPTION))
assorted.append(image.tag.get(TiffImagePlugin.STRIPOFFSETS))
assorted.append(image.tag.get(TiffImagePlugin.SAMPLESPERPIXEL))
assorted.append(image.tag.get(TiffImagePlugin.ROWSPERSTRIP))
assorted.append(image.tag.get(TiffImagePlugin.STRIPBYTECOUNTS))
assorted.append(image.tag.get(TiffImagePlugin.X_RESOLUTION))
assorted.append(image.tag.get(TiffImagePlugin.Y_RESOLUTION))
assorted.append(image.tag.get(TiffImagePlugin.PLANAR_CONFIGURATION))
assorted.append(image.tag.get(TiffImagePlugin.RESOLUTION_UNIT))
assorted.append(image.tag.get(TiffImagePlugin.SOFTWARE))
assorted.append(image.tag.get(TiffImagePlugin.DATE_TIME))
assorted.append(image.tag.get(TiffImagePlugin.ARTIST))
assorted.append(image.tag.get(TiffImagePlugin.PREDICTOR))
assorted.append(image.tag.get(TiffImagePlugin.COLORMAP))
assorted.append(image.tag.get(TiffImagePlugin.TILEOFFSETS))
assorted.append(image.tag.get(TiffImagePlugin.EXTRASAMPLES))
assorted.append(image.tag.get(TiffImagePlugin.SAMPLEFORMAT))
assorted.append(image.tag.get(TiffImagePlugin.JPEGTABLES))
assorted.append(image.tag.get(TiffImagePlugin.COPYRIGHT))
assorted.append(image.tag.get(TiffImagePlugin.IPTC_NAA_CHUNK))
assorted.append(image.tag.get(TiffImagePlugin.PHOTOSHOP_CHUNK))
assorted.append(image.tag.get(TiffImagePlugin.EXIFIFD))
# Delete variable
del image, info_dictionary
# Store the embedded XMP metadata
if config.ENABLEXMP:
import libxmp
xmpfile = libxmp.XMPFiles(file_path=fullpath)
assorted.append(str(xmpfile.get_xmp()))
xmpfile.close_file()
else:
assorted.append(None)
# Make sure we stored exactly the same amount of columns as
# specified!!
assert len(assorted) == len(columns)
# Fix date format
index = columns.index('datetime')
if assorted[index] is not None:
assorted[index] = dateutil.parser.parse(
assorted[index]).isoformat()
# Print some data that is stored in the database if debug is true
if config.DEBUG:
print "\nTiff file data:"
for i in range(0, len(assorted)):
print "%-18s %s" % (columns[i], assorted[i])
return assorted
except:
traceback.print_exc(file=sys.stderr)
# Store values in database so not the whole application crashes
return None
def process(file, config, rcontext, columns=None):
fullpath = file.fullpath
try:
# Try to parse JPG data
image = Image.open(fullpath, "r")
# Create an empty list
assorted = [
image.app, image.applist, image.bits, image.quantization,
image.icclist, image.huffman_dc, image.huffman_ac, image.layer,
image.tile
]
info_dictionary = image.info
# Check if jfif is in info dictionary, if so put it in our list
if "jfif" in info_dictionary:
assorted.append(info_dictionary["jfif"])
info_dictionary.pop("jfif")
else:
assorted.append(None)
# Check if icc_profile is in info dictionary,
# if so put it in our list
if "icc_profile" in info_dictionary:
assorted.append(info_dictionary["icc_profile"])
info_dictionary.pop("icc_profile")
else:
assorted.append(None)
# Check if jfif_version is in info dictionary,
# if so put it in our list
if "jfif_version" in image.info:
assorted.append(info_dictionary["jfif_version"])
info_dictionary.pop("jfif_version")
else:
assorted.append(None)
# Check if jfif_density is in info dictionary,
# if so put it in our list
if "jfif_density" in image.info:
assorted.append(info_dictionary["jfif_density"])
info_dictionary.pop("jfif_density")
else:
assorted.append(None)
# Check if jfif_unit is in info dictionary,
# if so put it in our list
if "jfif_unit" in image.info:
assorted.append(info_dictionary["jfif_unit"])
info_dictionary.pop("jfif_unit")
else:
assorted.append(None)
# Check if adobe is in info dictionary,
# if so put it in our list
if "adobe" in info_dictionary:
assorted.append(info_dictionary["adobe"])
info_dictionary.pop("adobe")
else:
assorted.append(None)
# Check if adobe_transform is in info dictionary,
# if so put it in our list
if "adobe_transform" in image.info:
assorted.append(info_dictionary["adobe_transform"])
info_dictionary.pop("adobe_transform")
else:
assorted.append(None)
# Check if progression is in info dictionary,
# if so put it in our list
if "progression" in info_dictionary:
assorted.append(info_dictionary["progression"])
info_dictionary.pop("progression")
else:
assorted.append(None)
# Check if gamma is in info dictionary,
# if so put it in our list
if "quality" in info_dictionary:
assorted.append(info_dictionary["quality"])
info_dictionary.pop("quality")
else:
assorted.append(None)
# Check if optimize is in info dictionary,
# if so put it in our list
if "optimize" in info_dictionary:
assorted.append(info_dictionary["optimize"][0])
info_dictionary.pop("optimize")
else:
assorted.append(None)
# Check if optimize is in info dictionary,
# if so put it in our list
if "progressive" in info_dictionary:
assorted.append(info_dictionary["progressive"])
info_dictionary.pop("progressive")
else:
assorted.append(None)
# Check if flashpix is in info dictionary, if so put it in our list
if "flashpix" in image.info:
assorted.append(info_dictionary["flashpix"])
info_dictionary.pop("flashpix")
else:
assorted.append(None)
# Check if dpi is in info dictionary, if so put it in our list
if "dpi" in info_dictionary:
assorted.append(info_dictionary["dpi"][0])
assorted.append(info_dictionary["dpi"][1])
info_dictionary.pop("dpi")
else:
assorted.append(None)
assorted.append(None)
# Check if exif is in info dictionary,
# if so decode and put in our list
if "exif" in info_dictionary:
gps_key_exists = False
# Get all keys and check if they are in the JPEG
for key in TAGS:
if key == 0x8825:
# In Key 0x8825 some GPS data is stored
gps_key_exists = True
for gpskey in GPSTAGS:
assorted.append(image._getexif().get(gpskey))
else:
assorted.append(image._getexif().get(key))
if gps_key_exists == False:
for gpskey in GPSTAGS:
assorted.append(None)
info_dictionary.pop("exif")
else:
for key in TAGS:
if key != 0x8825:
assorted.append(None)
for gpskey in GPSTAGS:
assorted.append(None)
# If there are still other values in the dict
# then put those in column
assorted.append(info_dictionary)
# Delete variable
del info_dictionary, image
# Store the embedded XMP metadata
if config.ENABLEXMP:
import libxmp
xmpfile = libxmp.XMPFiles(file_path=fullpath)
assorted.append(str(xmpfile.get_xmp()))
xmpfile.close_file()
else:
assorted.append(None)
# Make sure we stored exactly the same amount of columns as
# specified!!
assert len(assorted) == len(columns)
# Print some data that is stored in the database if debug is true
if config.DEBUG:
print "\nJPEG file data:"
for i in range(0, len(assorted)):
print "%-18s %s" % (columns[i], assorted[i])
print
return assorted
except:
traceback.print_exc(file=sys.stderr)
# Store values in database so not the whole application crashes
return None
