def _maybe_upload_file(
cls,
file: Optional[str] = None,
content: Optional[bytes] = None,
user_provided_file: Optional[str] = None,
check_if_file_exists: bool = True,
):
# Exactly one of `file` or `content` must be provided
if (file is None) == (content is None):
raise ValueError(
"Exactly one of `file` or `content` must be provided")
if content is None:
assert file is not None
with open(file, "rb") as f:
content = f.read()
if check_if_file_exists:
bytes = len(content)
matching_files = openai.File.find_matching_files(
name=user_provided_file or f.name,
bytes=bytes,
purpose="fine-tune")
if len(matching_files) > 0:
file_ids = [f["id"] for f in matching_files]
sys.stdout.write(
"Found potentially duplicated files with name '{name}', purpose 'fine-tune' and size {size} bytes\n"
.format(
name=os.path.basename(matching_files[0]["filename"]),
size=matching_files[0]["bytes"],
))
sys.stdout.write("\n".join(file_ids))
while True:
sys.stdout.write(
"\nEnter file ID to reuse an already uploaded file, or an empty string to upload this file anyway: "
)
inp = sys.stdin.readline().strip()
if inp in file_ids:
sys.stdout.write(
"Reusing already uploaded file: {id}\n".format(
id=inp))
return inp
elif inp == "":
break
else:
sys.stdout.write(
"File id '{id}' is not among the IDs of the potentially duplicated files\n"
.format(id=inp))
buffer_reader = BufferReader(content, desc="Upload progress")
resp = openai.File.create(
file=buffer_reader,
purpose="fine-tune",
user_provided_filename=user_provided_file or file,
)
sys.stdout.write("Uploaded file from {file}: {id}\n".format(
file=user_provided_file or file, id=resp["id"]))
return resp["id"]
def create(cls, args):
with open(args.file, "rb") as file_reader:
buffer_reader = BufferReader(file_reader.read(),
desc="Upload progress")
resp = openai.File.create(
file=buffer_reader,
purpose=args.purpose,
model=args.model,
user_provided_filename=args.file,
)
print(resp)
def request_raw(
self, method, url, params=None, supplied_headers=None, stream=False
):
"""
Mechanism for issuing an API call
"""
if self.api_key:
my_api_key = self.api_key
else:
from openai import api_key
my_api_key = api_key
if my_api_key is None:
raise error.AuthenticationError(
"No API key provided. (HINT: set your API key using in code using "
'"openai.api_key = <API-KEY>", or you can set the environment variable OPENAI_API_KEY=<API-KEY>). You can generate API keys '
"in the OpenAI web interface. See https://onboard.openai.com "
"for details, or email [email protected] if you have any "
"questions."
)
abs_url = "%s%s" % (self.api_base, url)
headers = {}
compress = None
progress_meter = False
if method == "get" or method == "delete":
if params:
encoded_params = url_encode_params(params)
abs_url = _build_api_url(abs_url, encoded_params)
else:
encoded_params = None
post_data = None
elif method in {"post", "put"}:
if (
supplied_headers is not None
and supplied_headers.get("Content-Type") == "multipart/form-data"
):
generator = MultipartDataGenerator()
generator.add_params(params or {})
post_data = generator.get_post_data()
content_type = "multipart/form-data; boundary=%s" % (
generator.boundary,
)
# We will overrite Content-Type
supplied_headers.pop("Content-Type")
progress_meter = True
# compress = "gzip"
compress = None
else:
post_data = json.dumps(params).encode()
content_type = "application/json"
headers["Content-Type"] = content_type
encoded_params = post_data
if progress_meter:
post_data = BufferReader(post_data, desc="Upload progress")
if compress == "gzip":
if not hasattr(post_data, "read"):
post_data = BytesIO(post_data)
headers["Content-Encoding"] = "gzip"
from openai.gzip_stream import GZIPCompressedStream
post_data = GZIPCompressedStream(post_data, compression_level=9)
else:
raise error.APIConnectionError(
"Unrecognized HTTP method %r. This may indicate a bug in the "
"OpenAI bindings. Please contact [email protected] for "
"assistance." % (method,)
)
headers = self.request_headers(my_api_key, method, headers)
if supplied_headers is not None:
for key, value in six.iteritems(supplied_headers):
headers[key] = value
util.log_info("Request to OpenAI API", method=method, path=abs_url)
util.log_debug(
"Post details", post_data=encoded_params, api_version=self.api_version
)
rbody, rcode, rheaders, stream = self._client.request_with_retries(
method, abs_url, headers, post_data, stream=stream
)
util.log_info(
"OpenAI API response",
path=abs_url,
response_code=rcode,
processing_ms=rheaders.get("OpenAI-Processing-Ms"),
)
util.log_debug("API response body", body=rbody, headers=rheaders)
if "Request-Id" in rheaders:
request_id = rheaders["Request-Id"]
util.log_debug(
"Dashboard link for request", link=util.dashboard_link(request_id)
)
return rbody, rcode, rheaders, stream, my_api_key