def get_contents(bucket, key, ext, *, etag, version_id, s3_client, size):
"""get the byte contents of a file"""
if ext.endswith('.gz'):
compression = 'gz'
ext = ext[:-len('.gz')]
else:
compression = None
content = ""
inferred_ext = infer_extensions(key, ext)
if inferred_ext in CONTENT_INDEX_EXTS:
if inferred_ext == ".ipynb":
content = trim_to_bytes(
# we have no choice but to fetch the entire notebook, because we
# are going to parse it
# warning: huge notebooks could spike memory here
get_notebook_cells(bucket,
key,
size,
compression,
etag=etag,
s3_client=s3_client,
version_id=version_id),
ELASTIC_LIMIT_BYTES)
elif inferred_ext == ".parquet":
if size >= get_available_memory():
print(
f"{bucket}/{key} too large to deserialize; skipping contents"
)
# at least index the key and other stats, but don't overrun memory
# and fail indexing altogether
return ""
obj = retry_s3("get",
bucket,
key,
size,
etag=etag,
s3_client=s3_client,
version_id=version_id)
body, info = extract_parquet(get_bytes(obj["Body"], compression),
as_html=False,
skip_rows=(inferred_ext
in SKIP_ROWS_EXTS))
# be smart and just send column names to ES (instead of bloated full schema)
# if this is not an HTML/catalog preview
columns = ','.join(list(info['schema']['names']))
content = trim_to_bytes(f"{columns}\n{body}", ELASTIC_LIMIT_BYTES)
else:
content = get_plain_text(bucket,
key,
size,
compression,
etag=etag,
s3_client=s3_client,
version_id=version_id)
return content
def maybe_get_contents(bucket, key, ext, *, etag, version_id, s3_client, size):
"""get the byte contents of a file if it's a target for deep indexing"""
logger_ = get_quilt_logger()
if ext.endswith('.gz'):
compression = 'gz'
ext = ext[:-len('.gz')]
else:
compression = None
logger_.debug(
"Entering maybe_get_contents (could run out of mem.) %s %s %s", bucket, key, version_id
)
content = ""
inferred_ext = infer_extensions(key, ext)
if inferred_ext in get_content_index_extensions(bucket_name=bucket):
def _get_obj():
return retry_s3(
"get",
bucket,
key,
size,
etag=etag,
s3_client=s3_client,
version_id=version_id,
)
if inferred_ext == ".fcs":
obj = _get_obj()
body, info = extract_fcs(get_bytes(obj["Body"], compression), as_html=False)
# be smart and just send column names to ES (instead of bloated full schema)
# if this is not an HTML/catalog preview
content = trim_to_bytes(f"{body}\n{info}", get_content_index_bytes(bucket_name=bucket))
elif inferred_ext == ".ipynb":
content = trim_to_bytes(
# we have no choice but to fetch the entire notebook, because we
# are going to parse it
# warning: huge notebooks could spike memory here
get_notebook_cells(
bucket,
key,
size,
compression,
etag=etag,
s3_client=s3_client,
version_id=version_id
),
get_content_index_bytes(bucket_name=bucket),
)
elif inferred_ext == ".parquet":
if size >= get_available_memory():
print(f"{bucket}/{key} too large to deserialize; skipping contents")
# at least index the key and other stats, but don't overrun memory
# and fail indexing altogether
return ""
obj = _get_obj()
body, info = extract_parquet(
get_bytes(obj["Body"], compression),
as_html=False,
skip_rows=(inferred_ext in SKIP_ROWS_EXTS),
max_bytes=get_content_index_bytes(bucket_name=bucket),
)
# be smart and just send column names to ES (instead of bloated full schema)
# if this is not an HTML/catalog preview
columns = ','.join(list(info['schema']['names']))
content = trim_to_bytes(f"{columns}\n{body}", get_content_index_bytes(bucket_name=bucket))
elif inferred_ext == ".pdf":
obj = _get_obj()
content = trim_to_bytes(
extract_pdf(get_bytes(obj["Body"], compression)),
get_content_index_bytes(bucket_name=bucket),
)
elif inferred_ext in (".xls", ".xlsx"):
obj = _get_obj()
body, _ = extract_excel(get_bytes(obj["Body"], compression), as_html=False)
content = trim_to_bytes(
body,
get_content_index_bytes(bucket_name=bucket),
)
elif inferred_ext == ".pptx":
obj = _get_obj()
content = extract_pptx(get_bytes(obj["Body"], compression), get_content_index_bytes(bucket_name=bucket))
else:
content = get_plain_text(
bucket,
key,
size,
compression,
etag=etag,
s3_client=s3_client,
version_id=version_id
)
return content