def makeDerivativeImages(self):
print "making derivative images"
hires_cmd = [
"ffmpeg", "-y", "-i", self.input,
os.path.join(self.output, "high_%s" % self.file_name)
]
hires = ShellThreader(hires_cmd)
hires.start()
hires.join()
resolutions.append({'thumb_': [0.15, 0.15]})
for resolution in resolutions:
label = resolution.keys()[0]
ffmpeg_cmd = [
"ffmpeg", "-y", "-i", self.input, "-vf",
"scale=iw*%.3f:ih*%.3f" %
(resolution[label][0], resolution[label][1]),
os.path.join(self.output, "%s%s" % (label, self.file_name))
]
print ffmpeg_cmd
ffmpeg = ShellThreader(ffmpeg_cmd)
ffmpeg.start()
ffmpeg.join()
def verifyVisualContent(input, ext_index, mime_type):
print "verifying visual content"
j = open("%s.j3m" % input[:ext_index], 'rb')
supplied_hashes = json.loads(j.read())['genealogy']['hashes']
j.close()
if mime_type == mime_types['image']:
verify = subprocess.call([
"java", "-jar",
"%s/packages/JavaMediaHasher/dist/JavaMediaHasher.jar" % main_dir,
input
],
stdout=open("%s.md5.txt" % input[:ext_index],
'wb+'))
md5 = open("%s.md5.txt" % input[:ext_index], 'rb')
verified_hash = md5.read().strip()
md5.close()
else:
verify = ShellThreader([
"ffmpeg", "-y", "-i", input, "-vcodec", "copy", "-an", "-f", "md5",
"%s.md5.txt" % input[:ext_index]
])
verify.start()
verify.join()
md5 = open("%s.md5.txt" % input[:ext_index], 'rb')
verified_hash = md5.read().strip().replace("MD5=", "")
md5.close()
print "comparing supplied %s hash with %s" % (supplied_hashes,
verified_hash)
if type(supplied_hashes) is list:
print "list of hashes..."
for hash in supplied_hashes:
print hash
if compareHash(hash, verified_hash):
return True
else:
return compareHash(supplied_hashes, verified_hash)
return False
def validateMediaObject(self, fileId, returnType=False):
super(ImportClient, self).validateMediaObject(fileId, returnType)
input = os.path.join(os.getcwd(), fileId)
output = "%s.log" % input[:-3]
ffmpeg_thread = ShellThreader([
"fab", "-f",
os.path.join("%sInformaCamData" % scripts_home['python'],
"ffmpeg_helper.py"),
"getInfo:input=%s,output=%s" % (input, output)
])
ffmpeg_thread.start()
ffmpeg_thread.join()
isValid = False
mime_type_found = None
input_0 = None
input_0_sentinel = "Input #0, "
for line in open(output, 'r'):
input_0_location = line.find(input_0_sentinel)
if input_0_location >= 0:
input_0 = line[(input_0_location + len(input_0_sentinel)):]
print input_0
if input_0.find("matroska") >= 0:
isValid = True
mime_type_found = self.mime_types['video']
break
elif input_0.find("image2") >= 0:
isValid = True
mime_type_found = self.mime_types['image']
break
os.remove(output)
if not returnType:
return isValid
else:
return mime_type_found
def getVideoMetadata(self):
"""
returns the j3m embedded in an .mkv
"""
print "getting video metadata"
# run ffmpeg command to dump result into a file called [fname].txt.b64
ffmpeg_cmd = [
"ffmpeg", "-y", "-dump_attachment:t",
"%s.txt.b64" % self.input[:self.ext_index], "-i", self.input
]
ffmpeg = ShellThreader(ffmpeg_cmd)
ffmpeg.start()
ffmpeg.join()
if self.getJ3MMetadata():
if not self.on_reindex:
p = Process(target=self.makeDerivativeVideos)
p.start()
return True
# json read that file and return object
return False
def __init__(self, submission):
from InformaCamModels.submission import Submission
from InformaCamModels.collection import Collection
from InformaCamModels.j3m import J3M
# unzip
self.asset_path = submission.asset_path.replace(
"/submissions/", "/collections/")
try:
os.makedirs(self.asset_path)
except OSError as e:
print e
print submission.emit()
unzip = ShellThreader([
"unzip",
"%s/%s" % (submission.asset_path, submission.file_name), "-d",
self.asset_path
])
unzip.start()
unzip.join()
print unzip.output
assets = []
media = []
sensor_captures = []
j3m_verified = False
j3m = None
rx = r'\s*inflating: (.*)'
for line in unzip.output:
#for line in unzip_output:
asset_match = re.findall(rx, line)
if len(asset_match) == 1:
assets.append(asset_match[0].strip())
r_j3m = r'.*/log.j3m'
r_caches = r'.*/informaCaches/.*'
for asset in assets:
j3m_match = re.match(r_j3m, asset)
if j3m_match is not None:
print "J3M: %s" % asset
j3m_verified = self.validateManifest()
j3m = J3M(path_to_j3m="%s.manifest" % asset)
cache_match = re.match(r_caches, asset)
if cache_match is not None:
print "ADD CACHE: %s" % asset
sensor_captures.append(asset)
else:
mime_type = getMimeTypeFromFile(asset)
print mime_type
if mime_type in [mime_types['image'], mime_types['video']]:
print "HERE IS A NEW SUB:%s" % asset
f = open(asset, 'rb')
data = {
'_id': hashString(asset),
'file_name': os.path.basename(asset),
'mime_type': mime_type,
'package_content': b64encode(f.read()).rstrip("="),
'sync_source': submission.sync_source
}
f.close()
if data['file_name'][
-4:] != ".%s" % mime_type_map[mime_type]:
data['file_name'] = "%s.%s" % (
data['file_name'], mime_type_map[mime_type])
try:
new_submission = Submission(inflate=data)
media.append(new_submission._id)
except exceptions.ConnectionError as e:
print e
return
inflate = {'_id': submission._id, 'asset_path': self.asset_path}
if len(media) > 0:
inflate['attached_media'] = media
if len(sensor_captures) > 0:
inflate['sensor_captures'] = sensor_captures
if j3m is not None:
inflate['j3m_id'] = j3m._id
inflate['dateCreated'] = j3m.genealogy['dateCreated']
collection = Collection(inflate=inflate)
print "NEW COLLECTION: %s" % collection._id
# delete self as submission
submission.delete()
def makeDerivativeVideos(self):
print "making derivative videos"
ffmpeg_cmd = [
"ffmpeg", "-y", "-i", self.input, "-vcodec", "copy", "-acodec",
"copy",
"%s.mp4" % self.input[:self.ext_index]
]
ffmpeg = ShellThreader(ffmpeg_cmd)
ffmpeg.start()
ffmpeg.join()
hires_cmd = [
"cp",
"%s.mp4" % self.input[:self.ext_index],
os.path.join(self.output,
"high_%s.mp4" % self.file_name[:self.ext_index])
]
hires = ShellThreader(hires_cmd)
hires.start()
hires.join()
print hires_cmd
ogv_cmd = [
"ffmpeg2theora",
"%s.mp4" % os.path.join(
self.output, "high_%s" % self.file_name[:self.ext_index])
]
print ogv_cmd
ogv = ShellThreader(ogv_cmd)
ogv.start()
ogv.join()
for resolution in resolutions:
label = resolution.keys()[0]
ffmpeg_cmd = [
"ffmpeg", "-y", "-i",
"%s.mp4" % self.input[:self.ext_index], "-filter:v",
"scale=%d:%d" % (resolution[label][0], resolution[label][1]),
"-acodec", "copy",
"%s.mp4" %
os.path.join(self.output, "%s%s" %
(label, self.file_name))[:self.ext_index]
]
ffmpeg = ShellThreader(ffmpeg_cmd)
ffmpeg.start()
ffmpeg.join()
ogv_cmd = [
"ffmpeg2theora",
"%s.mp4" %
os.path.join(self.output, "%s%s" %
(label, self.file_name))[:self.ext_index]
]
ogv = ShellThreader(ogv_cmd)
ogv.start()
ogv.join()
ffmpeg_cmd = [
"ffmpeg", "-y", "-i",
"%s.mp4" % self.input[:self.ext_index], "-f", "image2", "-ss",
"0.342", "-vframes", "1",
os.path.join(self.output,
"thumb_%s.jpg" % self.file_name[:self.ext_index])
]
ffmpeg = ShellThreader(ffmpeg_cmd)
ffmpeg.start()
ffmpeg.join()
def getJ3MMetadata(self):
"""
parses the j3m file
"""
print "parsing j3m data"
# un b64 [fname].txt.b64 and save as [fname].txt.unb64
b64 = open("%s.txt.b64" % self.input[:self.ext_index])
txt = open("%s.txt.unb64" % self.input[:self.ext_index], 'wb+')
content = b64.read()
try:
txt.write(base64.b64decode(content))
except TypeError as e:
print e
print "...so trying to decode again (brute-force padding)"
try:
content += "=" * ((4 - len(content) % 4) % 4)
txt.write(base64.b64decode(content))
except TypeError as e:
print e
print "could not unB64 this file (%s.txt.b64)" % self.input[:self
.
ext_index]
txt.close()
b64.close()
return False
txt.close()
b64.close()
m = magic.Magic(flags=magic.MAGIC_MIME_TYPE)
try:
file_type = m.id_filename("%s.txt.unb64" %
self.input[:self.ext_index])
print "DOC TYPE: %s" % file_type
m.close()
# if file is either JSON, PGP or GZIP
accepted_mime_types = [mime_types['pgp'], mime_types['gzip']]
if file_type not in accepted_mime_types:
print "NOT SUPPORTED FILE TYPE"
return False
if file_type == mime_types['pgp']:
print "ATTEMPTING TO DECRYPT BLOB"
pwd = open(gnupg_pword, 'rb')
passphrase = pwd.read().strip()
pwd.close()
print "read pwd"
gpg_cmd = [
"gpg",
"--no-tty",
"--passphrase",
passphrase,
"--output",
"%s.j3m.gz" % self.input[:self.ext_index],
"--decrypt",
"%s.txt.unb64" % self.input[:self.ext_index],
]
gpg_thread = ShellThreader(gpg_cmd)
gpg_thread.start()
gpg_thread.join()
print gpg_thread.output
# check to see if this new output is a gzip
gpg_check = magic.Magic(flags=magic.MAGIC_MIME_TYPE)
try:
file_type = gpg_check.id_filename(
"%s.j3m.gz" % self.input[:self.ext_index])
print "NEW DOC TYPE: %s" % file_type
except:
print "STILL FAILED TO DECRYPT"
gpg_check.close()
return False
gpg_check.close()
if file_type not in [mime_types['gzip'], mime_types['zip']]:
print "NO, this is still wrong."
return False
if file_type == mime_types['zip']:
# it is a j3m log!
print "WE HAVE A J3M LOG!"
os.rename("%s.j3m.gz" % self.input[:self.ext_index],
"%s.j3m.zip" % self.input[:self.ext_index])
self.submission.file_name = "%s.j3m.zip" % self.submission.file_name[:self
.
ext_index]
from j3mlogger import J3MLogger
J3MLogger(self.submission)
return
elif file_type == mime_types['gzip']:
# this was already a gzip; skip
print "THIS WAS ALREADY A GZIP: %s.txt.unb64" % self.input[:self
.
ext_index]
os.rename("%s.txt.unb64" % self.input[:self.ext_index],
"%s.j3m.gz" % self.input[:self.ext_index])
except:
m.close()
return False
# un gzip [fname].j3m.gz and save as [fname].j3m.orig
j3m = open("%s.j3m.orig" % self.input[:self.ext_index], 'wb+')
j3m.write(unGzipAsset("%s.j3m.gz" % self.input[:self.ext_index]))
j3m.close()
# if [fname].j3m.orig is text/plain and is json-readable
m = magic.Magic(flags=magic.MAGIC_MIME_TYPE)
try:
file_type = m.id_filename("%s.j3m.orig" %
self.input[:self.ext_index])
m.close()
except:
m.close()
return False
if file_type != mime_types['j3m']:
return False
f = open("%s.j3m.orig" % self.input[:self.ext_index], 'rb')
j3m_data = f.read()
f.close()
# extract signature as [fname].json.sig
try:
j3m_sig = json.loads(j3m_data)['signature']
except KeyError as e:
return False
f = open("%s.j3m.sig" % self.input[:self.ext_index], 'wb+')
f.write(j3m_sig)
f.close()
# save to [fname].j3m
front_sentinel = "{\"j3m\":"
back_sentinel = ",\"signature\":"
# this must be the LAST INSTANCE OF signature, btw.
print "FINDING BACK SENTINEL at position %d" % j3m_data.rindex(
back_sentinel)
extracted_j3m = j3m_data[len(front_sentinel):j3m_data.
rindex(back_sentinel)]
f = open("%s.j3m" % self.input[:self.ext_index], 'wb+')
f.write(extracted_j3m)
f.close()
j3m_asset = J3M(path_to_j3m="%s.j3m" % self.input[:self.ext_index])
self.submission.j3m_id = j3m_asset._id
self.submission.save()
return True
