def put_reference(self, name): if name not in self.get_references(name): put_binary_on_file( self.get_reference_path(name), NEW_LINE_AS_BYTES.join([to_bytes(name), b'']), mode='ab', retries=self.retries, retry_errno=self.retry_errno)
def __call__(self, data): data = to_bytes(data) for size, func in self.special_func: mimetype = func(data[:size]) if mimetype: return mimetype for required_size, key, mimetype in self.startswith_func: if data[:required_size].startswith(key): return mimetype
def add_startswith(self, key, mimetype): key = to_bytes(key) size = len(key) for i, (s, k, m) in enumerate(self.startswith_func): if s < size: self.startswith_func.insert(i, (size, key, mimetype)) break else: self.startswith_func.append((size, key, mimetype)) first_size = self.startswith_func[0][0] if first_size > self.min_header_size: self.min_header_size = first_size
def file_unique_code(open_file, block_size=OPEN_BLOCK_SIZE): current_position = open_file.tell() h = sha256() open_file.seek(0) block = open_file.read(block_size) convert_to_bytes = bool(not isinstance(open_file, BytesIO) and 'b' not in open_file.mode) while block: if convert_to_bytes: block = to_bytes(block) h.update(block) block = open_file.read(block_size) open_file.seek(current_position) return h.hexdigest()
def compress_images(self): month_str = TODAY_DATE().strftime('%Y%m') if not self.api_session_manager.tinypng_api or month_str in self.api_session_manager.tinypng_locked_months: return None headers = { 'Authorization': b'Basic ' + b64encode(b'api:' + to_bytes(self.api_session_manager.tinypng_api))} while True: file_info = ( self.session .query(File.id, File.file_id, File.filename) .filter(FilePath.compressed.is_(False)) .filter(FilePath.mimetype.in_(['image/jpeg', 'image/png'])) .filter(FilePath.id == File.file_id) .order_by(FilePath.size, File.filename.desc()) .first()) if not file_info: break binary = self.get_file_binary(file_info.id).read() response = open_json_url( 'https://api.tinify.com/shrink', method='POST', data=binary, headers=headers) self.api_session_manager.tinypng_locked_months.append(month_str) if response['output']['ratio'] >= 1: self.direct_update(FilePath, FilePath.id == file_info.file_id, {'compressed': True}) else: new_file_binary = get_url_file(response['output']['url'], headers=headers) file_path_id = self.save_file_path(new_file_binary, filename=file_info.filename, compressed=True).id if file_path_id == file_info.file_id: self.direct_update(FilePath, FilePath.id == file_info.file_id, {'compressed': True}) else: self.direct_update(File, File.file_id == file_info.file_id, {'file_id': file_path_id}) self.delete_file_paths(file_info.file_id)
def open_url(url, data=None, timeout=None, headers=None, method='get'): if timeout: timeout = abs(float(timeout)) url = to_string(url) if ' ' in url: url = url.replace(' ', '%20') if data: if isinstance(data, string_types): data = to_string(data) elif isinstance(data, (dict, MultiDict)): data = list(data.items()) if isinstance(data, (dict, tuple, list)): data = dict((to_string(k), to_string(v)) for k, v in data) data = urlencode(data) if method.lower() == 'get': url += '?%s' % data data = None else: data = to_bytes(data) req = Request(url, data=data, headers=headers or {}) req.get_method = lambda: method.upper() try: response = URL_OPENER.open(req, timeout=timeout) except Exception as error: message = u('Could not open the url: %s') % to_unicode(url) raise Error('url', message, exception=error) if isinstance(response, inesHTTPError): raise Error('url', response.message, exception=response) else: return response
def put_binary_on_file( path, binary, mode='wb', retries=3, retry_errno=DEFAULT_RETRY_ERRNO, make_dir_recursively=False): if 'b' in mode: binary = to_bytes(binary) else: binary = to_string(binary) try: with open(path, mode) as f: f.write(binary) except IOError as error: if error.errno is errno.ENOENT: # Missing folder, create and try again make_dir(dirname(path), make_dir_recursively=make_dir_recursively) elif error.errno not in retry_errno: raise else: # Try again, or not! retries -= 1 if not retries: raise return put_binary_on_file( path, binary, mode=mode, retries=retries, retry_errno=retry_errno, make_dir_recursively=make_dir_recursively) else: return True
def extend_values(self, name, values, expire=MARKER): if not values: raise ValueError('Define some values') binary = b_linesep.join(to_bytes(v) for v in values) + b_linesep self.put_binary(name, binary, mode='append', expire=expire)
def format_name(self, name): return to_bytes(make_sha256(name))
from ines.convert import maybe_set from ines.convert import string_join from ines.convert import to_bytes from ines.locks import LockMe from ines.locks import LockMeMemcached from ines.path import join_paths from ines.utils import file_modified_time from ines.utils import get_file_binary from ines.utils import make_dir from ines.utils import make_uuid_hash from ines.utils import move_file from ines.utils import put_binary_on_file from ines.utils import remove_file_quietly b_linesep = to_bytes(linesep) class _SaveMe(object): def get_binary(self, name, expire=MARKER): pass def put_binary(self, name, binary, mode='put', expire=MARKER): pass def __delitem__(self, name): pass def __contains__(self, name): pass
def string_unique_code(value): return sha256(to_bytes(value)).hexdigest()
def format_error_response_to_json(*args, **kwargs): return to_bytes(format_error_to_json(*args, **kwargs))
def format_position_name(self, name_256, position): name = '%s.%s' % (name_256, position) return to_bytes(make_sha256(name))
def format_name(self, name): return to_bytes(make_sha256('locks %s' % name))
def string_unique_code(value): value = to_bytes(value) return to_unicode(sha256(value).hexdigest())
def dumps(self, appstruct): return to_bytes(appstruct)