def upload_notebook(notebook, session=None):
"""Uploads a notebook to S3 in the default SageMaker Python SDK bucket for
this user. The resulting S3 object will be named "s3://<bucket>/papermill-input/notebook-YYYY-MM-DD-hh-mm-ss.ipynb".
Args:
notebook (str):
The filename of the notebook you want to upload. (Required)
session (boto3.Session):
A boto3 session to use. Will create a default session if not supplied. (Default: None)
Returns:
The resulting object name in S3 in URI format.
"""
session = ensure_session(session)
s3 = session.client("s3")
bucket = default_bucket(session)
prefix = f"papermill_input/{time.strftime('%Y-%m-%d-%H-%M-%S', time.gmtime())}"
directory, nb_filename = os.path.split(notebook)
for root, dirs, files in os.walk(directory, followlinks=True):
for filename in files:
local_path = os.path.join(root, filename)
relative_path = os.path.relpath(local_path, directory)
s3_path = os.path.join(prefix, relative_path)
try:
s3.head_object(Bucket=bucket, Key=s3_path)
except:
s3.upload_file(local_path, bucket, s3_path)
return f"s3://{bucket}/{prefix}/"
def save_csv_to_s3(df, csv_name):
session = ensure_session()
df.to_csv(csv_name, index=False)
s3 = session.client("s3")
bucket = default_bucket(session)
prefix = "full_repo_scan"
s3_path = os.path.join(prefix, csv_name)
s3.upload_file(csv_name, bucket, s3_path)
return f"s3://{bucket}/{prefix}/{csv_name}"
def save_csv_to_s3(notebooks, job_names, kernels):
session = ensure_session()
df = pd.DataFrame({
"filename": notebooks,
"processing-job-name": job_names,
"kernel": kernels
})
csv_name = f"{time.strftime('%Y-%m-%d-%H-%M-%S', time.gmtime())}.csv"
df.to_csv(csv_name, index=False)
s3 = session.client("s3")
bucket = default_bucket(session)
prefix = "full_repo_scan"
s3_path = os.path.join(prefix, csv_name)
try:
s3.head_object(Bucket=bucket, Key=s3_path)
except:
s3.upload_file(csv_name, bucket, s3_path)
return f"s3://{bucket}/{prefix}/{csv_name}"
def upload_fileobj(notebook_fileobj, session=None):
"""Uploads a file object to S3 in the default SageMaker Python SDK bucket for
this user. The resulting S3 object will be named "s3://<bucket>/papermill-input/notebook-YYYY-MM-DD-hh-mm-ss.ipynb".
Args:
notebook_fileobj (fileobj):
A file object (as returned from open) that is reading from the notebook you want to upload. (Required)
session (boto3.Session):
A boto3 session to use. Will create a default session if not supplied. (Default: None)
Returns:
The resulting object name in S3 in URI format.
"""
session = ensure_session(session)
snotebook = f"notebook-{time.strftime('%Y-%m-%d-%H-%M-%S', time.gmtime())}.ipynb"
s3 = session.client("s3")
key = "papermill_input/" + snotebook
bucket = default_bucket(session)
s3path = f"s3://{bucket}/{key}"
s3.upload_fileobj(notebook_fileobj, bucket, key)
return s3path