def put_reference(self, name):
if name not in self.get_references(name):
put_binary_on_file(
self.get_reference_path(name),
NEW_LINE_AS_BYTES.join([to_bytes(name), b'']),
mode='ab',
retries=self.retries,
retry_errno=self.retry_errno)
def __call__(self, data):
data = to_bytes(data)
for size, func in self.special_func:
mimetype = func(data[:size])
if mimetype:
return mimetype
for required_size, key, mimetype in self.startswith_func:
if data[:required_size].startswith(key):
return mimetype
def add_startswith(self, key, mimetype):
key = to_bytes(key)
size = len(key)
for i, (s, k, m) in enumerate(self.startswith_func):
if s < size:
self.startswith_func.insert(i, (size, key, mimetype))
break
else:
self.startswith_func.append((size, key, mimetype))
first_size = self.startswith_func[0][0]
if first_size > self.min_header_size:
self.min_header_size = first_size
def file_unique_code(open_file, block_size=OPEN_BLOCK_SIZE):
current_position = open_file.tell()
h = sha256()
open_file.seek(0)
block = open_file.read(block_size)
convert_to_bytes = bool(not isinstance(open_file, BytesIO) and 'b' not in open_file.mode)
while block:
if convert_to_bytes:
block = to_bytes(block)
h.update(block)
block = open_file.read(block_size)
open_file.seek(current_position)
return h.hexdigest()
def compress_images(self):
month_str = TODAY_DATE().strftime('%Y%m')
if not self.api_session_manager.tinypng_api or month_str in self.api_session_manager.tinypng_locked_months:
return None
headers = {
'Authorization': b'Basic ' + b64encode(b'api:' + to_bytes(self.api_session_manager.tinypng_api))}
while True:
file_info = (
self.session
.query(File.id, File.file_id, File.filename)
.filter(FilePath.compressed.is_(False))
.filter(FilePath.mimetype.in_(['image/jpeg', 'image/png']))
.filter(FilePath.id == File.file_id)
.order_by(FilePath.size, File.filename.desc())
.first())
if not file_info:
break
binary = self.get_file_binary(file_info.id).read()
response = open_json_url(
'https://api.tinify.com/shrink',
method='POST',
data=binary,
headers=headers)
self.api_session_manager.tinypng_locked_months.append(month_str)
if response['output']['ratio'] >= 1:
self.direct_update(FilePath, FilePath.id == file_info.file_id, {'compressed': True})
else:
new_file_binary = get_url_file(response['output']['url'], headers=headers)
file_path_id = self.save_file_path(new_file_binary, filename=file_info.filename, compressed=True).id
if file_path_id == file_info.file_id:
self.direct_update(FilePath, FilePath.id == file_info.file_id, {'compressed': True})
else:
self.direct_update(File, File.file_id == file_info.file_id, {'file_id': file_path_id})
self.delete_file_paths(file_info.file_id)
def open_url(url, data=None, timeout=None, headers=None, method='get'):
if timeout:
timeout = abs(float(timeout))
url = to_string(url)
if ' ' in url:
url = url.replace(' ', '%20')
if data:
if isinstance(data, string_types):
data = to_string(data)
elif isinstance(data, (dict, MultiDict)):
data = list(data.items())
if isinstance(data, (dict, tuple, list)):
data = dict((to_string(k), to_string(v)) for k, v in data)
data = urlencode(data)
if method.lower() == 'get':
url += '?%s' % data
data = None
else:
data = to_bytes(data)
req = Request(url, data=data, headers=headers or {})
req.get_method = lambda: method.upper()
try:
response = URL_OPENER.open(req, timeout=timeout)
except Exception as error:
message = u('Could not open the url: %s') % to_unicode(url)
raise Error('url', message, exception=error)
if isinstance(response, inesHTTPError):
raise Error('url', response.message, exception=response)
else:
return response
def put_binary_on_file(
path,
binary,
mode='wb',
retries=3,
retry_errno=DEFAULT_RETRY_ERRNO,
make_dir_recursively=False):
if 'b' in mode:
binary = to_bytes(binary)
else:
binary = to_string(binary)
try:
with open(path, mode) as f:
f.write(binary)
except IOError as error:
if error.errno is errno.ENOENT:
# Missing folder, create and try again
make_dir(dirname(path), make_dir_recursively=make_dir_recursively)
elif error.errno not in retry_errno:
raise
else:
# Try again, or not!
retries -= 1
if not retries:
raise
return put_binary_on_file(
path,
binary,
mode=mode,
retries=retries,
retry_errno=retry_errno,
make_dir_recursively=make_dir_recursively)
else:
return True
def extend_values(self, name, values, expire=MARKER):
if not values:
raise ValueError('Define some values')
binary = b_linesep.join(to_bytes(v) for v in values) + b_linesep
self.put_binary(name, binary, mode='append', expire=expire)
def format_name(self, name):
return to_bytes(make_sha256(name))
from ines.convert import maybe_set
from ines.convert import string_join
from ines.convert import to_bytes
from ines.locks import LockMe
from ines.locks import LockMeMemcached
from ines.path import join_paths
from ines.utils import file_modified_time
from ines.utils import get_file_binary
from ines.utils import make_dir
from ines.utils import make_uuid_hash
from ines.utils import move_file
from ines.utils import put_binary_on_file
from ines.utils import remove_file_quietly
b_linesep = to_bytes(linesep)
class _SaveMe(object):
def get_binary(self, name, expire=MARKER):
pass
def put_binary(self, name, binary, mode='put', expire=MARKER):
pass
def __delitem__(self, name):
pass
def __contains__(self, name):
pass
def string_unique_code(value):
return sha256(to_bytes(value)).hexdigest()
def format_error_response_to_json(*args, **kwargs):
return to_bytes(format_error_to_json(*args, **kwargs))
def format_position_name(self, name_256, position):
name = '%s.%s' % (name_256, position)
return to_bytes(make_sha256(name))
def format_name(self, name):
return to_bytes(make_sha256('locks %s' % name))
def string_unique_code(value):
value = to_bytes(value)
return to_unicode(sha256(value).hexdigest())
def dumps(self, appstruct):
return to_bytes(appstruct)