Exemplo n.º 1
0
 def put_reference(self, name):
     if name not in self.get_references(name):
         put_binary_on_file(
             self.get_reference_path(name),
             NEW_LINE_AS_BYTES.join([to_bytes(name), b'']),
             mode='ab',
             retries=self.retries,
             retry_errno=self.retry_errno)
Exemplo n.º 2
0
    def __call__(self, data):
        data = to_bytes(data)
        for size, func in self.special_func:
            mimetype = func(data[:size])
            if mimetype:
                return mimetype

        for required_size, key, mimetype in self.startswith_func:
            if data[:required_size].startswith(key):
                return mimetype
Exemplo n.º 3
0
    def add_startswith(self, key, mimetype):
        key = to_bytes(key)
        size = len(key)
        for i, (s, k, m) in enumerate(self.startswith_func):
            if s < size:
                self.startswith_func.insert(i, (size, key, mimetype))
                break
        else:
            self.startswith_func.append((size, key, mimetype))

        first_size = self.startswith_func[0][0]
        if first_size > self.min_header_size:
            self.min_header_size = first_size
Exemplo n.º 4
0
def file_unique_code(open_file, block_size=OPEN_BLOCK_SIZE):
    current_position = open_file.tell()

    h = sha256()
    open_file.seek(0)
    block = open_file.read(block_size)
    convert_to_bytes = bool(not isinstance(open_file, BytesIO) and 'b' not in open_file.mode)

    while block:
        if convert_to_bytes:
            block = to_bytes(block)

        h.update(block)
        block = open_file.read(block_size)

    open_file.seek(current_position)
    return h.hexdigest()
Exemplo n.º 5
0
    def compress_images(self):
        month_str = TODAY_DATE().strftime('%Y%m')
        if not self.api_session_manager.tinypng_api or month_str in self.api_session_manager.tinypng_locked_months:
            return None

        headers = {
            'Authorization': b'Basic ' + b64encode(b'api:' + to_bytes(self.api_session_manager.tinypng_api))}

        while True:
            file_info = (
                self.session
                .query(File.id, File.file_id, File.filename)
                .filter(FilePath.compressed.is_(False))
                .filter(FilePath.mimetype.in_(['image/jpeg', 'image/png']))
                .filter(FilePath.id == File.file_id)
                .order_by(FilePath.size, File.filename.desc())
                .first())

            if not file_info:
                break

            binary = self.get_file_binary(file_info.id).read()
            response = open_json_url(
                'https://api.tinify.com/shrink',
                method='POST',
                data=binary,
                headers=headers)

            self.api_session_manager.tinypng_locked_months.append(month_str)

            if response['output']['ratio'] >= 1:
                self.direct_update(FilePath, FilePath.id == file_info.file_id, {'compressed': True})
            else:
                new_file_binary = get_url_file(response['output']['url'], headers=headers)
                file_path_id = self.save_file_path(new_file_binary, filename=file_info.filename, compressed=True).id
                if file_path_id == file_info.file_id:
                    self.direct_update(FilePath, FilePath.id == file_info.file_id, {'compressed': True})
                else:
                    self.direct_update(File, File.file_id == file_info.file_id, {'file_id': file_path_id})
                    self.delete_file_paths(file_info.file_id)
Exemplo n.º 6
0
def open_url(url, data=None, timeout=None, headers=None, method='get'):
    if timeout:
        timeout = abs(float(timeout))

    url = to_string(url)
    if ' ' in url:
        url = url.replace(' ', '%20')

    if data:
        if isinstance(data, string_types):
            data = to_string(data)
        elif isinstance(data, (dict, MultiDict)):
            data = list(data.items())

        if isinstance(data, (dict, tuple, list)):
            data = dict((to_string(k), to_string(v)) for k, v in data)
            data = urlencode(data)

        if method.lower() == 'get':
            url += '?%s' % data
            data = None
        else:
            data = to_bytes(data)

    req = Request(url, data=data, headers=headers or {})
    req.get_method = lambda: method.upper()

    try:
        response = URL_OPENER.open(req, timeout=timeout)
    except Exception as error:
        message = u('Could not open the url: %s') % to_unicode(url)
        raise Error('url', message, exception=error)

    if isinstance(response, inesHTTPError):
        raise Error('url', response.message, exception=response)
    else:
        return response
Exemplo n.º 7
0
def put_binary_on_file(
        path,
        binary,
        mode='wb',
        retries=3,
        retry_errno=DEFAULT_RETRY_ERRNO,
        make_dir_recursively=False):

    if 'b' in mode:
        binary = to_bytes(binary)
    else:
        binary = to_string(binary)

    try:
        with open(path, mode) as f:
            f.write(binary)
    except IOError as error:
        if error.errno is errno.ENOENT:
            # Missing folder, create and try again
            make_dir(dirname(path), make_dir_recursively=make_dir_recursively)
        elif error.errno not in retry_errno:
            raise
        else:
            # Try again, or not!
            retries -= 1
            if not retries:
                raise

        return put_binary_on_file(
            path,
            binary,
            mode=mode,
            retries=retries,
            retry_errno=retry_errno,
            make_dir_recursively=make_dir_recursively)
    else:
        return True
Exemplo n.º 8
0
    def extend_values(self, name, values, expire=MARKER):
        if not values:
            raise ValueError('Define some values')

        binary = b_linesep.join(to_bytes(v) for v in values) + b_linesep
        self.put_binary(name, binary, mode='append', expire=expire)
Exemplo n.º 9
0
 def format_name(self, name):
     return to_bytes(make_sha256(name))
Exemplo n.º 10
0
from ines.convert import maybe_set
from ines.convert import string_join
from ines.convert import to_bytes
from ines.locks import LockMe
from ines.locks import LockMeMemcached
from ines.path import join_paths
from ines.utils import file_modified_time
from ines.utils import get_file_binary
from ines.utils import make_dir
from ines.utils import make_uuid_hash
from ines.utils import move_file
from ines.utils import put_binary_on_file
from ines.utils import remove_file_quietly


b_linesep = to_bytes(linesep)


class _SaveMe(object):
    def get_binary(self, name, expire=MARKER):
        pass

    def put_binary(self, name, binary, mode='put', expire=MARKER):
        pass

    def __delitem__(self, name):
        pass

    def __contains__(self, name):
        pass
Exemplo n.º 11
0
def string_unique_code(value):
    return sha256(to_bytes(value)).hexdigest()
Exemplo n.º 12
0
def format_error_response_to_json(*args, **kwargs):
    return to_bytes(format_error_to_json(*args, **kwargs))
Exemplo n.º 13
0
 def format_position_name(self, name_256, position):
     name = '%s.%s' % (name_256, position)
     return to_bytes(make_sha256(name))
Exemplo n.º 14
0
 def format_name(self, name):
     return to_bytes(make_sha256('locks %s' % name))
Exemplo n.º 15
0
def string_unique_code(value):
    value = to_bytes(value)
    return to_unicode(sha256(value).hexdigest())
Exemplo n.º 16
0
 def dumps(self, appstruct):
     return to_bytes(appstruct)