def fetch_or_create(cls, data, file_type=None, im=None, attributes=()):
sha256 = hashlib.sha256(data).hexdigest()
obj = (cls.query.filter(cls.sha256 == sha256).first())
if obj is None:
attributes = dict(attributes)
if file_type is None and im is not None:
file_type = images.image_file_type(im)
if file_type is None:
raise ValueError('a file type is required')
if im is not None:
attributes.update({
'width': im.size.width,
'height': im.size.height
})
elif file_type == 'swf':
attributes.update(flash.parse_flash_header(BytesIO(data)))
obj = cls(sha256=sha256,
file_type=file_type,
attributes=attributes)
# Write our file to disk
real_path = obj.full_file_path
makedirs_exist_ok(os.path.dirname(real_path))
with open(real_path, 'wb') as outfile:
outfile.write(data)
cls.dbsession.add(obj)
return obj
def test_read_uncompressed_flash_header():
"""
parse_flash_header can parse the header of uncompressed flash files.
"""
infile = datadir.join('test.swf').open(mode='rb')
header = flash.parse_flash_header(infile)
assert header == {
'compression': None,
'size': 153,
'version': 5,
'width': 550,
'height': 400,
}
def test_read_lzma_compressed_flash_header():
"""
parse_flash_header can parse the header of lzma compressed flash files.
"""
infile = datadir.join('lzma.swf').open(mode='rb')
header = flash.parse_flash_header(infile)
assert header == {
'compression': 'lzma',
'size': 3294,
'version': 25,
'width': 550,
'height': 400,
}
def test_read_zlib_compressed_flash_header():
"""
parse_flash_header can parse the header of zlib compressed flash files.
"""
infile = datadir.join('flash_eyes.swf').open(mode='rb')
header = flash.parse_flash_header(infile)
assert header == {
'compression': 'zlib',
'size': 15900,
'version': 6,
'width': 300,
'height': 300,
}
def test_read_uncompressed_flash_header():
"""
parse_flash_header can parse the header of uncompressed flash files.
"""
infile = datadir.join('test.swf').open(mode='rb')
header = flash.parse_flash_header(infile)
assert header == {
'compression': None,
'size': 153,
'version': 5,
'width': 550,
'height': 400,
}
def test_read_lzma_compressed_flash_header():
"""
parse_flash_header can parse the header of lzma compressed flash files.
"""
infile = datadir.join('lzma.swf').open(mode='rb')
header = flash.parse_flash_header(infile)
assert header == {
'compression': 'lzma',
'size': 3294,
'version': 25,
'width': 550,
'height': 400,
}
def test_read_zlib_compressed_flash_header():
"""
parse_flash_header can parse the header of zlib compressed flash files.
"""
infile = datadir.join('flash_eyes.swf').open(mode='rb')
header = flash.parse_flash_header(infile)
assert header == {
'compression': 'zlib',
'size': 15900,
'version': 6,
'width': 300,
'height': 300,
}
def fetch_or_create(cls, data, file_type=None, im=None, attributes=()):
sha256 = hashlib.sha256(data).hexdigest()
obj = (cls.query
.filter(cls.sha256 == sha256)
.first())
if obj is None:
attributes = dict(attributes)
if file_type is None and im is not None:
file_type = images.image_file_type(im)
if file_type is None:
raise ValueError('a file type is required')
if im is not None:
attributes.update({'width': im.size.width, 'height': im.size.height})
elif file_type == 'swf':
attributes.update(flash.parse_flash_header(BytesIO(data)))
obj = cls(sha256=sha256, file_type=file_type, attributes=attributes)
obj.init_from_data(data)
cls.dbsession.add(obj)
return obj
def fetch_or_create(cls, data, file_type=None, im=None, attributes=()):
sha256 = hashlib.sha256(data).hexdigest()
obj = (cls.query.filter(cls.sha256 == sha256).first())
if obj is None:
attributes = dict(attributes)
if file_type is None and im is not None:
file_type = images.image_file_type(im)
if file_type is None:
raise ValueError('a file type is required')
if im is not None:
attributes.update({
'width': im.size.width,
'height': im.size.height
})
elif file_type == 'swf':
attributes.update(flash.parse_flash_header(BytesIO(data)))
obj = cls(sha256=sha256,
file_type=file_type,
attributes=attributes)
obj.init_from_data(data)
cls.dbsession.add(obj)
return obj
def main():
db = define.connect()
q = (db.query(
orm.MediaItem.mediaid).filter(orm.MediaItem.sha256 != None).filter(
orm.MediaItem.file_type == 'swf').order_by(
orm.MediaItem.sha256.asc()))
count = q.count()
for e, (mediaid, ) in enumerate(q, start=1):
sys.stderr.write('\r%d/%d' % (e, count))
m = db.query(orm.DiskMediaItem).get(mediaid)
try:
infile = open(m.full_file_path, 'rb')
except IOError as e:
if e.errno != errno.ENOENT:
raise
print "\rcouldn't locate", m.mediaid
else:
with infile:
m.attributes.update(flash.parse_flash_header(infile))
m.attributes.changed()
db.flush()
print
def main():
db = define.connect()
q = (
db.query(orm.MediaItem.mediaid)
.filter(orm.MediaItem.sha256 != None)
.filter(orm.MediaItem.file_type == 'swf')
.order_by(orm.MediaItem.sha256.asc()))
count = q.count()
for e, (mediaid,) in enumerate(q, start=1):
sys.stderr.write('\r%d/%d' % (e, count))
m = db.query(orm.DiskMediaItem).get(mediaid)
try:
infile = open(m.full_file_path, 'rb')
except IOError as e:
if e.errno != errno.ENOENT:
raise
print "\rcouldn't locate", m.mediaid
else:
with infile:
m.attributes.update(flash.parse_flash_header(infile))
m.attributes.changed()
db.flush()
print
