async def load(context, path):
file_path = join(context.config.FILE_LOADER_ROOT_PATH.rstrip("/"), path.lstrip("/"))
file_path = abspath(file_path)
inside_root_path = file_path.startswith(
abspath(context.config.FILE_LOADER_ROOT_PATH)
)
result = LoaderResult()
if not inside_root_path:
result.error = LoaderResult.ERROR_NOT_FOUND
result.successful = False
return result
# keep backwards compatibility, try the actual path first
# if not found, unquote it and try again
if not exists(file_path):
file_path = unquote(file_path)
if exists(file_path):
with open(file_path, "rb") as source_file:
stats = fstat(source_file.fileno())
result.successful = True
result.buffer = source_file.read()
result.metadata.update(
size=stats.st_size,
updated_at=datetime.utcfromtimestamp(stats.st_mtime),
)
else:
result.error = LoaderResult.ERROR_NOT_FOUND
result.successful = False
return result
def load(context, path, callback):
file_path = join(
context.config.FILE_LOADER_ROOT_PATH.rstrip('/'), path.lstrip('/'))
file_path = abspath(file_path)
inside_root_path = file_path.startswith(
abspath(context.config.FILE_LOADER_ROOT_PATH))
result = LoaderResult()
if not inside_root_path:
result.error = LoaderResult.ERROR_NOT_FOUND
result.successful = False
callback(result)
return
# keep backwards compatibility, try the actual path first
# if not found, unquote it and try again
if not exists(file_path):
file_path = unquote(file_path)
if exists(file_path):
with open(file_path, 'r') as f:
stats = fstat(f.fileno())
result.successful = True
result.buffer = f.read()
result.metadata.update(
size=stats.st_size,
updated_at=datetime.utcfromtimestamp(stats.st_mtime))
else:
result.error = LoaderResult.ERROR_NOT_FOUND
result.successful = False
callback(result)
def return_contents(response, url, callback, context, req_start=None):
if req_start:
finish = datetime.datetime.now()
res = urlparse(url)
context.metrics.timing(
'original_image.fetch.{0}.{1}'.format(response.code, res.netloc),
(finish - req_start).total_seconds() * 1000
)
result = LoaderResult()
context.metrics.incr('original_image.status.' + str(response.code))
if response.error:
result.successful = False
if response.code == 599:
# Return a Gateway Timeout status downstream if upstream times out
result.error = LoaderResult.ERROR_TIMEOUT
else:
result.error = LoaderResult.ERROR_NOT_FOUND
logger.warn(u"ERROR retrieving image {0}: {1}".format(url, str(response.error)))
elif response.body is None or len(response.body) == 0:
result.successful = False
result.error = LoaderResult.ERROR_UPSTREAM
logger.warn(u"ERROR retrieving image {0}: Empty response.".format(url))
else:
if response.time_info:
for x in response.time_info:
context.metrics.timing('original_image.time_info.' + x, response.time_info[x] * 1000)
context.metrics.timing('original_image.time_info.bytes_per_second', len(response.body) / response.time_info['total'])
result.buffer = response.body
context.metrics.incr('original_image.response_bytes', len(response.body))
callback(result)
def load_file(context, file_path, callback):
inside_root_path = file_path.startswith(
abspath(context.config.FILE_LOADER_ROOT_PATH))
result = LoaderResult()
if not inside_root_path:
result.error = LoaderResult.ERROR_NOT_FOUND
result.successful = False
callback(result)
return
# keep backwards compatibility, try the actual path first
# if not found, unquote it and try again
if not exists(file_path):
file_path = unquote(file_path)
if exists(file_path):
with open(file_path, 'r') as f:
stats = fstat(f.fileno())
result.successful = True
result.buffer = f.read()
result.metadata.update(
size=stats.st_size,
updated_at=datetime.datetime.utcfromtimestamp(stats.st_mtime))
else:
result.error = LoaderResult.ERROR_NOT_FOUND
result.successful = False
callback(result)
def load(context, url, callback, normalize_url_func=_normalize_url):
result = LoaderResult()
start = time.perf_counter()
try:
result.buffer = ffmpeg(context, normalize_url_func(url))
except subprocess.CalledProcessError as err:
result.successful = False
result.error = err.stderr.decode('utf-8').strip()
logger.warn(f'ERROR retrieving image {url}: {result.error}')
if result.error.lower().endswith(
'Server returned 404 not found'.lower()):
result.error = LoaderResult.ERROR_NOT_FOUND
except Exception as err:
result.successful = False
result.error = str(err)
logger.warn(f'ERROR retrieving image {url}: {err}')
else:
total_time = (time.perf_counter() - start)
total_bytes = len(result.buffer)
result.metadata.update({
'size': total_bytes,
# 'updated_at': datetime.datetime.utcnow(),
})
context.metrics.incr('original_image.status.200')
context.metrics.incr('original_image.response_bytes', total_bytes)
context.metrics.timing(f'original_image.fetch.{url}',
total_time * 1000)
context.metrics.timing('original_image.time_info.bytes_per_second',
total_bytes / total_time)
return callback(result)
def callback_wrapper(result):
r = LoaderResult()
if result is not None:
r.successful = True
r.buffer = result
else:
r.error = LoaderResult.ERROR_NOT_FOUND
r.successful = False
callback(r)
def _process_output(callback, destination_name, status):
result = LoaderResult()
if status != 0:
result.successful = False
else:
result.successful = True
with open(destination_name, 'rb') as f:
result.buffer = f.read()
os.remove(destination_name)
callback(result)
def _process_output(callback, destination_name, status):
result = LoaderResult()
if status != 0:
result.successful = False
else:
result.successful = True
with open(destination_name, 'rb') as f:
result.buffer = f.read()
os.remove(destination_name)
callback(result)
def return_contents(response, url, context, req_start=None):
res = urlparse(url)
if req_start:
finish = datetime.datetime.now()
context.metrics.timing(
"original_image.fetch.{0}.{1}".format(response.code,
res.netloc.replace(".",
"_")),
(finish - req_start).total_seconds() * 1000,
)
context.metrics.incr("original_image.fetch.{0}.{1}".format(
response.code, res.netloc.replace(".", "_")))
result = LoaderResult()
context.metrics.incr("original_image.status." + str(response.code))
context.metrics.incr("original_image.status.{0}.{1}".format(
response.code, res.netloc.replace(".", "_")))
if response.error:
result.successful = False
if response.code == 599:
# Return a Gateway Timeout status downstream if upstream times out
result.error = LoaderResult.ERROR_TIMEOUT
else:
result.error = LoaderResult.ERROR_NOT_FOUND
logger.warning(u"ERROR retrieving image %s: %s", url,
str(response.error))
elif response.body is None or len(response.body) == 0:
result.successful = False
result.error = LoaderResult.ERROR_UPSTREAM
logger.warning(u"ERROR retrieving image %s: Empty response.", url)
else:
if response.time_info:
for metric_name in response.time_info:
context.metrics.timing(
"original_image.time_info." + metric_name,
response.time_info[metric_name] * 1000,
)
context.metrics.timing(
"original_image.time_info.bytes_per_second",
len(response.body) // response.time_info["total"],
)
result.buffer = response.body
result.metadata.update(response.headers)
context.metrics.incr("original_image.response_bytes",
len(response.body))
return result
def load(self, path, callback):
db = __conn__(self)
words2 = path.split("/")
storage = self.config.MONGO_ORIGIN_SERVER_COLLECTION
images = gridfs.GridFS(db, collection=storage)
result = LoaderResult()
if images.exists(ObjectId(words2[0])):
contents = images.get(ObjectId(words2[0])).read()
result.successful = True
result.buffer = contents
else:
result.error = LoaderResult.ERROR_NOT_FOUND
result.successful = False
callback(result)
def load(context, path, callback):
result = LoaderResult()
conn = boto.connect_s3(
aws_access_key_id = context.config.AWS_S3_ACCESS_KEY,
aws_secret_access_key = context.config.AWS_S3_SECRET_KEY,
host=context.config.AWS_HOST,
calling_format = boto.s3.connection.OrdinaryCallingFormat(),
)
bucket = conn.get_bucket(context.config.AWS_S3_BUCKET,validate=False)
data = bucket.get_key(path)
if data is None:
result.successful = False
result.error = LoaderResult.ERROR_NOT_FOUND
else:
result.buffer = data.get_contents_as_string()
result.successful = True
callback(result)
def load(context, path, callback):
file_path = abspath(join(dirname(__file__), "fixtures/images/image.jpg"))
result = LoaderResult()
if exists(file_path):
with open(file_path, 'r') as f:
stats = fstat(f.fileno())
result.successful = True
result.buffer = f.read()
result.metadata.update(
size=stats.st_size,
updated_at=datetime.utcfromtimestamp(stats.st_mtime)
)
else:
result.error = LoaderResult.ERROR_NOT_FOUND
result.successful = False
callback(result)
def _process_done(
callback,
process,
context,
normalized_url,
seek,
output_file,
status
):
# T183907 Sometimes ffmpeg returns status 0 and actually fails to
# generate a thumbnail. We double-check the existence of the thumbnail
# in case of apparent success
if status == 0:
if os.stat(output_file.name).st_size == 0:
status = -1
# If rendering the desired frame fails, attempt to render the
# first frame instead
if status != 0 and seek > 0:
seek_and_screenshot(callback, context, normalized_url, 0)
return
result = LoaderResult()
if status != 0: # pragma: no cover
_http_code_from_stderr(context, process, result, normalized_url)
else:
result.successful = True
result.buffer = output_file.read()
process.stdout.close()
process.stderr.close()
output_file.close()
try:
os.unlink(output_file.name)
except OSError as e: # pragma: no cover
if e.errno != errno.ENOENT:
logger.error('[Video] Unable to unlink output file', extra=log_extra(context))
raise
callback(result)
def load(context, path, callback):
result = LoaderResult()
for idx, next_dir in enumerate(context.config.TC_MULTIDIR_PATHS):
file_path = join(next_dir.rstrip('/'), path.lstrip('/'))
file_path = abspath(file_path)
inside_root_path = file_path.startswith(abspath(next_dir))
if inside_root_path:
# keep backwards compatibility, try the actual path first
# if not found, unquote it and try again
found = exists(file_path)
if not found:
file_path = unquote(file_path)
found = exists(file_path)
if found:
with open(file_path, 'rb') as f:
stats = fstat(f.fileno())
result.successful = True
result.buffer = f.read()
result.metadata.update(
size=stats.st_size,
updated_at=datetime.utcfromtimestamp(stats.st_mtime))
callback(result)
return
logger.debug('TC_MULTIDIR: File {0} not found in {1}'.format(path, next_dir))
# else loop and try next directory
if not context.config.TC_MULTIDIR_PATHS:
logger.error('TC_MULTIDIR: No paths set in configuration TC_MULTIDIR_PATHS')
# no file found
result.error = LoaderResult.ERROR_NOT_FOUND
result.successful = False
callback(result)
def load(context, path, callback):
file_path = join(context.config.FILE_LOADER_ROOT_PATH.rstrip('/'), path.lstrip('/'))
file_path = abspath(file_path)
inside_root_path = file_path.startswith(context.config.FILE_LOADER_ROOT_PATH)
result = LoaderResult()
if inside_root_path and exists(file_path):
with open(file_path, 'r') as f:
stats = fstat(f.fileno())
result.successful = True
response = f.read()
excerpt_length = context.config.LOADER_EXCERPT_LENGTH
result.buffer = response[:excerpt_length]
if len(result.buffer) == excerpt_length:
temp = NamedTemporaryFile(delete=False)
temp.write(response)
temp.close()
context.wikimedia_original_file = temp
tornado.ioloop.IOLoop.instance().call_later(
context.config.HTTP_LOADER_TEMP_FILE_TIMEOUT,
partial(
cleanup_temp_file,
context.wikimedia_original_file.name
)
)
result.metadata.update(
size=stats.st_size,
updated_at=datetime.utcfromtimestamp(stats.st_mtime)
)
else:
result.error = 404
result.successful = False
callback(result)
def return_contents(response, url, callback, context, req_start=None):
if req_start:
finish = datetime.datetime.now()
res = urlparse(url)
context.metrics.timing(
'original_image.fetch.{0}.{1}'.format(response.code, res.netloc),
(finish - req_start).total_seconds() * 1000,
)
result = LoaderResult()
context.metrics.incr('original_image.status.' + str(response.code))
if response.error:
result.successful = False
if response.code == 599:
# Return a Gateway Timeout status downstream if upstream times out
result.error = LoaderResult.ERROR_TIMEOUT
else:
result.error = LoaderResult.ERROR_NOT_FOUND
logger.warn(u"ERROR retrieving image {0}: {1}".format(
url, str(response.error)))
elif response.body is None or len(response.body) == 0:
result.successful = False
result.error = LoaderResult.ERROR_UPSTREAM
logger.warn(u"ERROR retrieving image {0}: Empty response.".format(url))
else:
if response.time_info:
for x in response.time_info:
context.metrics.timing('original_image.time_info.' + x,
response.time_info[x] * 1000)
context.metrics.timing(
'original_image.time_info.bytes_per_second',
len(response.body) / response.time_info['total'])
result.buffer = response.body
result.metadata.update(response.headers)
context.metrics.incr('original_image.response_bytes', len(
response.body))
callback(result)
def return_contents(response, url, callback, context, req_start=None):
if req_start:
finish = datetime.datetime.now()
res = urlparse(url)
context.metrics.timing(
'original_image.fetch.{0}.{1}'.format(response.code, res.netloc),
(finish - req_start).total_seconds() * 1000,
)
result = LoaderResult()
context.metrics.incr('original_image.status.' + str(response.code))
if response.error:
result.successful = False
if response.code == 599:
# Return a Gateway Timeout status downstream if upstream times out
result.error = LoaderResult.ERROR_TIMEOUT
else:
result.error = LoaderResult.ERROR_NOT_FOUND
logger.warn(u"ERROR retrieving image {0}: {1}".format(
url, str(response.error)))
elif response.body is None or len(response.body) == 0:
result.successful = False
result.error = LoaderResult.ERROR_UPSTREAM
logger.warn(u"ERROR retrieving image {0}: Empty response.".format(url))
else:
if response.time_info:
for x in response.time_info:
context.metrics.timing('original_image.time_info.' + x,
response.time_info[x] * 1000)
context.metrics.timing(
'original_image.time_info.bytes_per_second',
len(response.body) / response.time_info['total'])
result.buffer = response.body
result.metadata.update(response.headers)
context.metrics.incr('original_image.response_bytes',
len(response.body))
if context.config.NORMALIZE_TO_72DPI and context.config.CONVERT_PATH:
if not exists(context.config.CONVERT_PATH):
logger.warn(
'imagemagick/convert enabled but binary CONVERT_PATH does not exist'
)
else:
try:
if "jpeg" in result.metadata['Content-Type']:
command = [
context.config.CONVERT_PATH +
' - -density 72,72 -strip - ',
]
normalize_dpi_cmd = Popen(command,
stdin=PIPE,
stdout=PIPE,
stderr=PIPE,
close_fds=True,
shell=True)
normalize_dpi_stdout, normalize_dpi_stderr = normalize_dpi_cmd.communicate(
input=response.body)
if normalize_dpi_cmd.returncode != 0:
logger.warn(
'dpi normalization finished with non-zero return code (%d): %s'
% (normalize_dpi_cmd.returncode,
normalize_dpi_stderr))
else:
result.buffer = normalize_dpi_stdout
except KeyError:
logger.warn('Content-Type KeyError on %s' % url)
callback(result)
def load(context, path):
file_path = join(context.config.PIC_LOADER_ROOT_PATH.rstrip('/'),
path.lstrip('/'))
file_path = abspath(file_path)
file_path_two = join(context.config.PIC_LOADER_FALLBACK_PATH.rstrip('/'),
path.lstrip('/'))
file_path_two = abspath(file_path_two)
inside_root_path = file_path.startswith(
abspath(context.config.PIC_LOADER_ROOT_PATH))
inside_root_path_two = file_path_two.startswith(
abspath(context.config.PIC_LOADER_FALLBACK_PATH))
if not exists(context.config.PIC_LOADER_MAX_SIZE):
oversize = context.config.PIC_LOADER_MAX_SIZE
else:
oversize = 16777216
result = LoaderResult()
if not inside_root_path:
if not inside_root_path_two:
result.error = LoaderResult.ERROR_NOT_FOUND
result.successful = False
return result #callback(result)
pass #return
# keep backwards compatibility, try the actual path first
# if not found, unquote it and try again
if not exists(file_path):
file_path = unquote(file_path)
if not exists(file_path_two):
file_path_two = unquote(file_path_two)
if exists(file_path):
with open(file_path, 'r') as f:
stats = fstat(f.fileno())
if stats.st_size >= oversize:
result.error = LoaderResult.COULD_NOT_LOAD_IMAGE
result.successful = False
else:
result.successful = True
result.buffer = f.read()
result.metadata.update(size=stats.st_size,
updated_at=datetime.utcfromtimestamp(
stats.st_mtime))
elif exists(file_path_two):
with open(file_path_two, 'r') as f:
stats = fstat(f.fileno())
if stats.st_size >= oversize:
result.error = LoaderResult.COULD_NOT_LOAD_IMAGE
result.successful = False
else:
result.successful = True
result.buffer = f.read()
result.metadata.update(size=stats.st_size,
updated_at=datetime.utcfromtimestamp(
stats.st_mtime))
else:
result.error = LoaderResult.ERROR_NOT_FOUND
result.successful = False
return result
def load(context, path, callback):
file_path = join(context.config.PIC_LOADER_ROOT_PATH.rstrip('/'),
path.lstrip('/'))
file_path = abspath(file_path)
file_path_two = join(context.config.PIC_LOADER_FALLBACK_PATH.rstrip('/'),
path.lstrip('/'))
file_path_two = abspath(file_path_two)
inside_root_path = file_path.startswith(
abspath(context.config.PIC_LOADER_ROOT_PATH))
inside_root_path_two = file_path_two.startswith(
abspath(context.config.PIC_LOADER_FALLBACK_PATH))
result = LoaderResult()
if not inside_root_path:
if not inside_root_path_two:
result.error = LoaderResult.ERROR_NOT_FOUND
result.successful = False
callback(result)
return
# keep backwards compatibility, try the actual path first
# if not found, unquote it and try again
if not exists(file_path):
file_path = unquote(file_path)
if not exists(file_path_two):
file_path_two = unquote(file_path_two)
if exists(file_path):
with open(file_path, 'r') as f:
stats = fstat(f.fileno())
if stats.st_size <= 1:
logger.warning(u"%s: cette image source est vide...",
file_path)
result.successful = False
result.error = LoaderResult.ERROR_UPSTREAM
else:
result.successful = True
result.buffer = f.read()
result.metadata.update(size=stats.st_size,
updated_at=datetime.utcfromtimestamp(
stats.st_mtime))
elif exists(file_path_two):
with open(file_path_two, 'r') as f:
stats = fstat(f.fileno())
if stats.st_size <= 4:
logger.warning(u"%s: cette image source est vide...",
file_path_two)
result.successful = False
result.error = LoaderResult.ERROR_UPSTREAM
else:
result.successful = True
result.buffer = f.read()
result.metadata.update(size=stats.st_size,
updated_at=datetime.utcfromtimestamp(
stats.st_mtime))
else:
result.error = LoaderResult.ERROR_NOT_FOUND
result.successful = False
callback(result)
def load_sync(context, url, callback):
logger.debug('[SWIFT_LOADER] load_sync: %s' % url, extra=log_extra(context))
result = LoaderResult()
container = context.wikimedia_original_container
path = context.wikimedia_original_filepath
try:
logger.debug(
'[SWIFT_LOADER] fetching %s from container %s' % (path, container),
extra=log_extra(context)
)
start = datetime.datetime.now()
logging.disable(logging.ERROR)
headers, response = swift(context).get_object(
container,
path
)
logging.disable(logging.NOTSET)
record_timing(context, datetime.datetime.now() - start, 'swift.original.read.success', 'Thumbor-Swift-Original-Success-Time')
context.metrics.incr('swift_loader.status.success')
# XXX hack: If the file is an STL, we overwrite the first five bytes
# with the word "solid", to trick the MIME detection pipeline.
extension = path[-4:].lower()
isSTL = extension == '.stl'
f = NamedTemporaryFile(delete=False)
logger.debug(
'[SWIFT_LOADER] writing %d bytes to temp file' % len(response),
extra=log_extra(context)
)
f.write(response)
f.close()
excerpt_length = context.config.LOADER_EXCERPT_LENGTH
# First kb of the body for MIME detection
body = response[:excerpt_length]
# See above - text STLs have this string here anyway, and
# binary STLs ignore the first 80 bytes, so this string will
# be ignored.
if isSTL:
body = 'solid' + body[5:]
if len(body) == excerpt_length:
logger.debug('[SWIFT_LOADER] return_contents: %s' % f.name, extra=log_extra(context))
context.wikimedia_original_file = f
tornado.ioloop.IOLoop.instance().call_later(
context.config.HTTP_LOADER_TEMP_FILE_TIMEOUT,
partial(
cleanup_temp_file,
context,
context.wikimedia_original_file.name
)
)
else:
logger.debug('[SWIFT_LOADER] return_contents: small body')
cleanup_temp_file(context, f.name)
result.buffer = body
except ClientException as e:
record_timing(context, datetime.datetime.now() - start, 'swift.original.read.miss', 'Thumbor-Swift-Original-Miss-Time')
logging.disable(logging.NOTSET)
result.successful = False
result.error = LoaderResult.ERROR_NOT_FOUND
logger.error('[SWIFT_LOADER] get_object failed: %s %r' % (url, e), extra=log_extra(context))
context.metrics.incr('swift_loader.status.client_exception')
except requests.ConnectionError as e:
record_timing(context, datetime.datetime.now() - start, 'swift.original.read.exception', 'Thumbor-Swift-Original-Exception-Time')
logging.disable(logging.NOTSET)
result.successful = False
result.error = LoaderResult.ERROR_UPSTREAM
logger.error('[SWIFT_LOADER] get_object failed: %s %r' % (url, e), extra=log_extra(context))
context.metrics.incr('swift_loader.status.connection_error')
callback(result)
