def handle_uploaded_dataset_file_internal( filename, datatypes_registry, ext='auto', tmp_prefix='sniff_upload_', tmp_dir=None, in_place=False, check_content=True, is_binary=None, auto_decompress=True, uploaded_file_ext=None, convert_to_posix_lines=None, convert_spaces_to_tabs=None, ): is_valid, ext, converted_path, compressed_type = handle_compressed_file( filename, datatypes_registry, ext=ext, tmp_prefix=tmp_prefix, tmp_dir=tmp_dir, in_place=in_place, check_content=check_content, auto_decompress=auto_decompress, ) try: if not is_valid: if is_tar(converted_path): raise InappropriateDatasetContentError('TAR file uploads are not supported') raise InappropriateDatasetContentError('The uploaded compressed file contains invalid content') # This needs to be checked again after decompression is_binary = check_binary(converted_path) if not is_binary and (convert_to_posix_lines or convert_spaces_to_tabs): # Convert universal line endings to Posix line endings, spaces to tabs (if desired) if convert_spaces_to_tabs: convert_fxn = convert_newlines_sep2tabs else: convert_fxn = convert_newlines line_count, _converted_path = convert_fxn(converted_path, in_place=in_place, tmp_dir=tmp_dir, tmp_prefix=tmp_prefix) if not in_place: if converted_path and filename != converted_path: os.unlink(converted_path) converted_path = _converted_path if ext in AUTO_DETECT_EXTENSIONS: ext = guess_ext(converted_path, sniff_order=datatypes_registry.sniff_order, is_binary=is_binary) if not is_binary and check_content and check_html(converted_path): raise InappropriateDatasetContentError('The uploaded file contains invalid HTML content') except Exception: if filename != converted_path: os.unlink(converted_path) raise return ext, converted_path, compressed_type
def handle_uploaded_dataset_file_internal( filename, datatypes_registry, ext='auto', tmp_prefix='sniff_upload_', tmp_dir=None, in_place=False, check_content=True, is_binary=None, auto_decompress=True, uploaded_file_ext=None, convert_to_posix_lines=None, convert_spaces_to_tabs=None, ): is_valid, ext, converted_path, compressed_type = handle_compressed_file( filename, datatypes_registry, ext=ext, tmp_prefix=tmp_prefix, tmp_dir=tmp_dir, in_place=in_place, check_content=check_content, auto_decompress=auto_decompress, ) try: if not is_valid: if is_tar(converted_path): raise InappropriateDatasetContentError('TAR file uploads are not supported') raise InappropriateDatasetContentError('The uploaded compressed file contains invalid content') # This needs to be checked again after decompression is_binary = check_binary(converted_path) if not is_binary and (convert_to_posix_lines or convert_spaces_to_tabs): # Convert universal line endings to Posix line endings, spaces to tabs (if desired) if convert_spaces_to_tabs: convert_fxn = convert_newlines_sep2tabs else: convert_fxn = convert_newlines line_count, _converted_path = convert_fxn(converted_path, in_place=in_place, tmp_dir=tmp_dir, tmp_prefix=tmp_prefix) if not in_place: if converted_path and filename != converted_path: os.unlink(converted_path) converted_path = _converted_path if ext in AUTO_DETECT_EXTENSIONS: ext = guess_ext(converted_path, sniff_order=datatypes_registry.sniff_order, is_binary=is_binary) if not is_binary and check_content and check_html(converted_path): raise InappropriateDatasetContentError('The uploaded file contains invalid HTML content') except Exception: if filename != converted_path: os.unlink(converted_path) raise return ext, converted_path, compressed_type
def handle_uploaded_dataset_file_internal( filename: str, datatypes_registry, ext: str = 'auto', tmp_prefix: Optional[str] = 'sniff_upload_', tmp_dir: Optional[str] = None, in_place: bool = False, check_content: bool = True, is_binary: Optional[bool] = None, auto_decompress: bool = True, uploaded_file_ext: Optional[str] = None, convert_to_posix_lines: Optional[bool] = None, convert_spaces_to_tabs: Optional[bool] = None, ) -> HandleUploadedDatasetFileInternalResponse: is_valid, ext, converted_path, compressed_type = handle_compressed_file( filename, datatypes_registry, ext=ext, tmp_prefix=tmp_prefix, tmp_dir=tmp_dir, in_place=in_place, check_content=check_content, auto_decompress=auto_decompress, ) converted_newlines = False converted_spaces = False try: if not is_valid: if is_tar(converted_path): raise InappropriateDatasetContentError( 'TAR file uploads are not supported') raise InappropriateDatasetContentError( 'The uploaded compressed file contains invalid content') # This needs to be checked again after decompression is_binary = check_binary(converted_path) guessed_ext = ext if ext in AUTO_DETECT_EXTENSIONS: guessed_ext = guess_ext(converted_path, sniff_order=datatypes_registry.sniff_order, is_binary=is_binary) guessed_datatype = datatypes_registry.get_datatype_by_extension( guessed_ext) if not is_binary and guessed_datatype.is_binary: # It's possible to have a datatype that is binary but not within the first 1024 bytes, # so check_binary might return a false negative. This is for instance true for PDF files is_binary = True if not is_binary and (convert_to_posix_lines or convert_spaces_to_tabs): # Convert universal line endings to Posix line endings, spaces to tabs (if desired) convert_fxn = convert_function(convert_to_posix_lines, convert_spaces_to_tabs) line_count, _converted_path, converted_newlines, converted_spaces = convert_fxn( converted_path, in_place=in_place, tmp_dir=tmp_dir, tmp_prefix=tmp_prefix) if not in_place: if converted_path and filename != converted_path: os.unlink(converted_path) assert _converted_path converted_path = _converted_path if ext in AUTO_DETECT_EXTENSIONS: ext = guess_ext(converted_path, sniff_order=datatypes_registry.sniff_order, is_binary=is_binary) else: ext = guessed_ext if not is_binary and check_content and check_html(converted_path): raise InappropriateDatasetContentError( 'The uploaded file contains invalid HTML content') except Exception: if filename != converted_path: os.unlink(converted_path) raise return HandleUploadedDatasetFileInternalResponse(ext, converted_path, compressed_type, converted_newlines, converted_spaces)
def handle_uploaded_dataset_file_internal( filename, datatypes_registry, ext='auto', tmp_prefix='sniff_upload_', tmp_dir=None, in_place=False, check_content=True, is_binary=None, auto_decompress=True, uploaded_file_ext=None, convert_to_posix_lines=None, convert_spaces_to_tabs=None, ): is_valid, ext, converted_path, compressed_type = handle_compressed_file( filename, datatypes_registry, ext=ext, tmp_prefix=tmp_prefix, tmp_dir=tmp_dir, in_place=in_place, check_content=check_content, auto_decompress=auto_decompress, ) try: if not is_valid: if is_tar(converted_path): raise InappropriateDatasetContentError( 'TAR file uploads are not supported') raise InappropriateDatasetContentError( 'The uploaded compressed file contains invalid content') # This needs to be checked again after decompression is_binary = check_binary(converted_path) guessed_ext = ext if ext in AUTO_DETECT_EXTENSIONS: guessed_ext = guess_ext(converted_path, sniff_order=datatypes_registry.sniff_order, is_binary=is_binary) guessed_datatype = datatypes_registry.get_datatype_by_extension( guessed_ext) if not is_binary and guessed_datatype.is_binary: # It's possible to have a datatype that is binary but not within the first 1024 bytes, # so check_binary might return a false negative. This is for instance true for PDF files is_binary = True if not is_binary and (convert_to_posix_lines or convert_spaces_to_tabs): # Convert universal line endings to Posix line endings, spaces to tabs (if desired) if convert_spaces_to_tabs: convert_fxn = convert_newlines_sep2tabs else: convert_fxn = convert_newlines line_count, _converted_path = convert_fxn(converted_path, in_place=in_place, tmp_dir=tmp_dir, tmp_prefix=tmp_prefix) if not in_place: if converted_path and filename != converted_path: os.unlink(converted_path) converted_path = _converted_path if ext in AUTO_DETECT_EXTENSIONS: ext = guess_ext(converted_path, sniff_order=datatypes_registry.sniff_order, is_binary=is_binary) else: ext = guessed_ext # AMP Commenting this out. Does not seem necessary. check_html returns true for valid HTML, but the error # thrown is "invalid html". Similarly, if we change it to not check_html, then it throws an error with JSON # or other non-binary data types #if not is_binary and check_content and check_html(converted_path): #raise InappropriateDatasetContentError('The uploaded file contains invalid HTML content') except Exception: if filename != converted_path: os.unlink(converted_path) raise return ext, converted_path, compressed_type