def mediacache_audio(blob_urn, mime_type, additional_args, audio):
if additional_args:
return None
if blob_urn != audio.blob.href:
return None
if mime_type == audio.blob.mime_type:
return iter(audio.blob)
input_suffix = '.' + mime_to_ext[audio.blob.mime_type]
output_suffix = '.' + mime_to_ext[mime_type]
try:
cmd = AUDIO_CONVERSION_COMMANDS[(audio.blob.mime_type, mime_type)]
except KeyError:
return None # we don't support this converstion
else:
input_filename = iterator_to_tempfile(iter(audio.blob), suffix=input_suffix)
try:
tmpdir = mkdtemp()
def delete_tmpdir(filename=None):
rmtree(tmpdir, ignore_errors=True)
output_filename = os.path.join(tmpdir, "audio" + output_suffix)
logger.info("Converting %s to %s", input_filename, output_filename)
cmd = cmd.format(input_filename=input_filename,
output_filename=output_filename)
try:
subprocess.check_call(cmd, shell=True)
except Exception:
delete_tmpdir()
raise
return iterate_file_then_delete(output_filename, delete_func=delete_tmpdir)
finally:
os.remove(input_filename)
def __getitem__(self, key):
try:
return self.__cache[key]
except Exception:
data_iterator = self.__backing_store[key]
# Cache it
tmpfile = iterator_to_tempfile(data_iterator)
self.__attempt_cache_save(key, tmpfile)
return iterate_file_then_delete(tmpfile)
def store(self, data_iterator, urn=None):
"""data_iterator is an iterator that returns all data.
Returns the urn or could raise some exceptions."""
intended_urn = urn
data_iterator = (ensure_isinstance(data, bytes)
for data in data_iterator)
header, data_iterator = determine_header(data_iterator)
hash_obj = hash_algorithm()
data_iterator = check_resource_size(data_iterator,
self.max_resource_size)
data_iterator = calculate_hash(data_iterator, hash_obj)
tmpfile = iterator_to_tempfile(data_iterator)
try:
digest = hash_encode(hash_obj.digest()).decode("ascii")
urn = "urn:%s:%s" % (hash_name, digest)
if intended_urn and intended_urn != urn:
raise "URN given does not match content." # valueerror
# Do we already have this urn in the DB?
if urn in self.storage_backend:
# compare with what we have
with file(tmpfile, 'rb') as f:
for d in self.storage_backend[urn]:
dd = f.read(len(d))
if dd != d:
# Collision!? Save aside and raise exception
self.storage_backend.put_file(
'%s-collision' % urn, tmpfile)
raise Exception("hash collision")
return urn # new resource equals old one
# If it is an XML file, check it
if header == 'xml':
self.__check_xml(urn, tmpfile)
self.storage_backend.put_file(urn, tmpfile)
finally:
os.remove(tmpfile)
return urn
def __getitem__(self, key):
for i, backend in enumerate(self.__backends):
try:
data_iterator = backend[key]
except KeyError:
pass
else:
if self.collect_resources and i > 0:
# Save a copy to the primary backend
tmpfile = iterator_to_tempfile(data_iterator)
try:
primary_backend = self.__backends[0]
primary_backend.put_file(key, tmpfile)
except Exception:
pass
data_iterator = iterate_file_then_delete(tmpfile)
return data_iterator
raise KeyError(key)
def put(urn, mime_type, additional_args, data_iterator):
filename = __to_filename(urn, mime_type, additional_args)
dirname = os.path.dirname(filename)
try:
os.makedirs(dirname, mode=0755)
except OSError:
# fail only if the directory doesn't already exist
if not os.path.isdir(dirname):
raise
tmpfile = iterator_to_tempfile(data_iterator)
try:
copyfile(tmpfile, filename)
except OSError:
return False
finally:
os.remove(tmpfile)
return True
def wrapped_getitem(s, key):
data_iterator = original_getitem(s, key)
max_resource_size = getattr(s, "max_resource_size", (20*1024*1024))
data_iterator = check_resource_size(data_iterator, max_resource_size)
# Calculate hash and save to temporary file
hash_obj = hash_algorithm()
data_iterator = calculate_hash(data_iterator, hash_obj)
tmpfile = iterator_to_tempfile(data_iterator)
# Verify hash
try:
digest = hash_encode(hash_obj.digest())
if key != "urn:%s:%s" % (hash_name, digest):
raise "URN given does not match content." # valueerror
except:
os.remove(tmpfile)
raise
return iterate_file_then_delete(tmpfile)
def handle_noargs(self, **options):
# obtain the list of banned ip addresses
url = 'http://www.stopforumspam.com/downloads/bannedips.zip'
csv_zip = urlopen(url).read()
zipfile = ZipFile(StringIO(csv_zip))
filename = zipfile.namelist()[0]
csv_string = zipfile.read(filename)
banned_ips = csv_string.strip(',').split(',')
# rearrange things for efficient storage
banned_ips = sorted(inet_aton(b) for b in banned_ips)
banned_ips_bytes = b''.join(banned_ips)
# save the banned ip list to disk
output_filename = settings.DUCTUS_BLACKLIST_FILE
if not output_filename:
raise Exception
tmpfile = iterator_to_tempfile([banned_ips_bytes])
try:
copyfile(tmpfile, output_filename)
finally:
os.remove(tmpfile)
def handle_noargs(self, **options):
# obtain the list of banned ip addresses
url = 'http://www.stopforumspam.com/downloads/bannedips.zip'
csv_zip = urlopen(url).read()
zipfile = ZipFile(StringIO(csv_zip))
filename = zipfile.namelist()[0]
csv_string = zipfile.read(filename)
banned_ips = csv_string.strip(',').split(',')
# rearrange things for efficient storage
banned_ips = sorted(inet_aton(b) for b in banned_ips)
banned_ips_bytes = b''.join(banned_ips)
# save the banned ip list to disk
output_filename = settings.DUCTUS_BLACKLIST_FILE
if not output_filename:
raise Exception
tmpfile = iterator_to_tempfile([banned_ips_bytes])
try:
copyfile(tmpfile, output_filename)
finally:
os.remove(tmpfile)
def mediacache_audio(blob_urn, mime_type, additional_args, audio):
if additional_args:
return None
if blob_urn != audio.blob.href:
return None
if mime_type == audio.blob.mime_type:
return iter(audio.blob)
input_suffix = '.' + mime_to_ext[audio.blob.mime_type]
output_suffix = '.' + mime_to_ext[mime_type]
try:
cmd = AUDIO_CONVERSION_COMMANDS[(audio.blob.mime_type, mime_type)]
except KeyError:
return None # we don't support this converstion
else:
input_filename = iterator_to_tempfile(iter(audio.blob),
suffix=input_suffix)
try:
tmpdir = mkdtemp()
def delete_tmpdir(filename=None):
rmtree(tmpdir, ignore_errors=True)
output_filename = os.path.join(tmpdir, "audio" + output_suffix)
logger.info("Converting %s to %s", input_filename, output_filename)
cmd = cmd.format(input_filename=input_filename,
output_filename=output_filename)
try:
subprocess.check_call(cmd, shell=True)
except Exception:
delete_tmpdir()
raise
return iterate_file_then_delete(output_filename,
delete_func=delete_tmpdir)
finally:
os.remove(input_filename)