def run_glue_job_from_local_folder_template(local_base, s3_base_path, name, role, job_args = None, allocated_capacity = None, max_retries = None, max_concurrent_runs = None):
"""
Take a local folder layed out using our agreed folder spec, upload to s3, and run
job_args is a dictionary that is passed to the glue job when it is run on aws.
"""
local_base = _end_with_slash(local_base)
s3_base_path = _end_with_slash(s3_base_path)
# Create kwargs for job defintion these will be used in glue_create_job_defintion
job_def_kwargs = {}
job_def_kwargs['Name'] = name
job_def_kwargs['Role'] = role
if allocated_capacity is not None :
job_def_kwargs['AllocatedCapacity'] = allocated_capacity
if max_retries is not None :
job_def_kwargs['MaxRetries'] = max_retries
if max_concurrent_runs is not None :
job_def_kwargs['MaxConcurrentRuns'] = max_concurrent_runs
bucket, bucket_folder = s3_path_to_bucket_key(s3_base_path)
bucket_folder = bucket_folder + '/'
delete_folder_from_bucket(bucket, bucket_folder)
glue_job_folder_to_s3(local_base, s3_base_path)
job_spec = glue_folder_in_s3_to_job_spec(s3_base_path, **job_def_kwargs)
response = glue_client.create_job(**job_spec)
if job_args:
response = glue_client.start_job_run(JobName=name, Arguments = job_args)
else:
response = glue_client.start_job_run(JobName=name)
return response, job_spec
def all_glue_job_folders_to_s3(local_glue_jobs_dir, s3_glue_jobs_dir, include_folders = None, exclude_folders = None) :
"""
Iterate though all folders in the glue_job dir and upload them to a corresponsing
glue_job dir in s3. Each folder in local_glue_jobs_dir is uploaded using glue_job_folder_to_s3.
Provide list of folder glue_job folder names in include_folders and exclude_folders to include and exclude them from the upload.
"""
local_glue_jobs_dir = _end_with_slash(local_glue_jobs_dir)
# Do checks
if not (include_folders is None or isinstance(include_folders, list)) :
raise ValueError('include_folders must be a list or None')
if not (exclude_folders is None or isinstance(exclude_folders, list)) :
raise ValueError('exclude_folders must be a list or None')
if include_folders is not None and exclude_folders is not None :
if len(set(include_folders).intersection(set(exclude_folders))) != 0 :
raise ValueError('Some folders are listed in both include_folders and exclude_folders')
# Create list of folders
glue_job_folders = [d for d in os.listdir(local_glue_jobs_dir) if os.path.isdir(os.path.join(local_glue_jobs_dir, d)) and d[0] != '.']
if include_folders is not None :
test_include = [i in glue_job_folders for i in include_folders]
if not all(test_include) :
raise ValueError('One of the folders listed in include_folders does not exist in {}'.format(local_glue_jobs_dir))
else :
glue_job_folders = include_folders
if exclude_folders is not None :
glue_job_folders = [g for g in glue_job_folders if g not in exclude_folders]
s3_glue_jobs_dir = _end_with_slash(s3_glue_jobs_dir)
for glue_job in glue_job_folders :
glue_job_folder_to_s3(local_glue_jobs_dir + glue_job + '/', s3_glue_jobs_dir + glue_job + '/')
def upload_directory_to_s3(dir_path, s3_dir_parent_path, regex = ".+(\.sql|\.json|\.csv|\.txt|\.py|\.sh)$") :
# Make sure folder paths are correct
dir_path = _end_with_slash(dir_path)
s3_dir_parent_path = _end_with_slash(s3_dir_parent_path)
dir_path_prefix = '/'.join(dir_path.split('/')[:-2])
if dir_path_prefix != '' :
dir_path_prefix = dir_path_prefix + '/'
bucket, key = s3_path_to_bucket_key(s3_dir_parent_path)
for root, directories, filenames in os.walk('init'):
for filename in filenames:
f = os.path.join(root,filename)
if re.match(regex, f) :
path_out = upload_file_to_s3_from_path(f, bucket, key + f.replace(dir_path_prefix, ''))
def run_glue_job_from_s3_folder_template(s3_glue_job_folder, name, role, job_args = None, allocated_capacity = None, max_retries = None, max_concurrent_runs = None) :
s3_glue_job_folder = _end_with_slash(s3_glue_job_folder)
job_def_kwargs = {}
job_def_kwargs['Name'] = name
job_def_kwargs['Role'] = role
if allocated_capacity is not None :
job_def_kwargs['AllocatedCapacity'] = allocated_capacity
if max_retries is not None :
job_def_kwargs['MaxRetries'] = max_retries
if max_concurrent_runs is not None :
job_def_kwargs['MaxConcurrentRuns'] = max_concurrent_runs
bucket, bucket_folder = s3_path_to_bucket_key(s3_glue_job_folder)
job_spec = glue_folder_in_s3_to_job_spec(s3_glue_job_folder, **job_def_kwargs)
del_response = delete_job(name)
response = glue_client.create_job(**job_spec)
if job_args:
response = glue_client.start_job_run(JobName=name, Arguments = job_args)
else:
response = glue_client.start_job_run(JobName=name)
return response, job_spec
def get_glue_job_and_resources_from_s3(s3_base_path) :
s3_base_path = _end_with_slash(s3_base_path)
bucket, bucket_folder = s3_path_to_bucket_key(s3_base_path)
bucket_folder = bucket_folder[:-1]
shared_bucket_folder = '/'.join(bucket_folder.split('/')[:-1]) + '/shared_job_resources'
files_list = get_file_list_from_bucket(bucket, bucket_folder)
if "{}/job.py".format(bucket_folder) not in files_list:
raise ValueError("Cannot find job.py in the folder specified ({}), stopping".format(bucket_folder))
else:
job_path = "s3://{}/{}/job.py".format(bucket, bucket_folder)
try :
shared_files_list = get_file_list_from_bucket(bucket, shared_bucket_folder)
except :
shared_files_list = []
# Do py_resources
py_resources = [f for f in files_list if "/glue_py_resources/" in f]
py_shared_resources = [f for f in shared_files_list if "/glue_py_resources/" in f]
py_resources = py_resources + py_shared_resources
py_resources = ["s3://{}/{}".format(bucket, f) for f in py_resources]
py_resources = ",".join(py_resources)
# Do resources
resources = [f for f in files_list if "/glue_resources/" in f]
shared_resources = [f for f in shared_files_list if "/glue_resources/" in f]
resources = resources + shared_resources
resources = ["s3://{}/{}".format(bucket, f) for f in resources]
resources = ",".join(resources)
if " " in resources or " " in py_resources :
raise ValueError("The files in glue_resources and glue_py_resources must not have spaces in their filenames")
return (job_path, resources, py_resources)
def glue_folder_in_s3_to_job_spec(s3_base_path, **kwargs) :
"""
Given a set of files uploaded to s3 in a specific format, use them to create a glue job
"""
#Base path should be a folder. Ensure ends in "/"
# Otherwise listing the bucket could cause problems in e.g. the case there are two jobs, job_1 and job_12
s3_base_path = _end_with_slash(s3_base_path)
bucket, bucket_folder = s3_path_to_bucket_key(s3_base_path)
(job_path, resources, py_resources) = get_glue_job_and_resources_from_s3(s3_base_path)
kwargs["ScriptLocation"] = job_path
if resources != '':
kwargs["extra-files"] = resources
if py_resources != '':
kwargs["extra-py-files"] = py_resources
kwargs["TempDir"] = "s3://{}/{}/{}/temp_dir/".format(bucket, bucket_folder, kwargs["Name"])
job_spec = create_glue_job_definition(**kwargs)
return job_spec
def get_file_list_from_bucket(bucket, bucket_folder) :
bucket_folder = _end_with_slash(bucket_folder)
contents = s3_client.list_objects(Bucket=bucket, Prefix=bucket_folder)
files_list = [c["Key"] for c in contents["Contents"]]
return files_list
def glue_job_folder_to_s3(local_base, s3_base_path):
"""
Take a folder structure on local disk and transfer to s3.
Folder must be formatted as follows:
base dir
job.py
glue_py_resources/
zip and python files
zip_urls <- file containing urls of additional zip files e.g. on github
glue_resources/
txt, sql, json, or csv files
The folder name base dir will be in the folder s3_path_to_glue_jobs_folder
"""
local_base = _end_with_slash(local_base)
s3_base_path = _end_with_slash(s3_base_path)
base_dir_listing = os.listdir(local_base)
# Upload job
bucket, bucket_folder = s3_path_to_bucket_key(s3_base_path)
bucket_folder = bucket_folder[:-1]
# Check that there is at least a job.py in the given folder and then upload job if appropriate
if 'job.py' not in base_dir_listing :
if local_base.split('/')[-2] != 'shared_job_resources' :
raise ValueError("Could not find job.py in base directory provided ({}), stopping.\nOnly folder allowed to have no job.py is a folder named shared_job_resources".format(local_base))
else :
local_job_path = os.path.join(local_base, "job.py")
job_path = upload_file_to_s3_from_path(local_job_path, bucket, "{}/job.py".format(bucket_folder))
# Upload all the .py or .zip files in resources
# Check existence of folder, otherwise skip
resources_path = os.path.join(local_base, "glue_resources")
if os.path.isdir(resources_path):
resource_listing = os.listdir(os.path.join(local_base, 'glue_resources'))
regex = ".+(\.sql|\.json|\.csv|\.txt)$"
resource_listing = [f for f in resource_listing if re.match(regex, f)]
for f in resource_listing:
resource_local_path = os.path.join(local_base, "glue_resources", f)
path = upload_file_to_s3_from_path(resource_local_path, bucket, "{}/glue_resources/{}".format(bucket_folder,f))
# Upload all the .py or .zip files in resources
# Check existence of folder, otherwise skip
py_resources_path = os.path.join(local_base, "glue_py_resources")
delete_these_paths = []
if os.path.isdir(py_resources_path):
zip_urls_path = os.path.join(py_resources_path, "github_zip_urls.txt")
if os.path.exists(zip_urls_path):
with open(zip_urls_path, "r") as f:
urls = f.readlines()
urls = [url for url in urls if len(url) > 10]
for i, url in enumerate(urls):
this_zip_path = os.path.join(py_resources_path,"{}.zip".format(i))
urlretrieve(url,this_zip_path)
new_zip_path = unnest_github_zipfile_and_return_new_zip_path(this_zip_path)
os.remove(this_zip_path)
delete_these_paths.append(new_zip_path)
resource_listing = os.listdir(os.path.join(local_base, 'glue_py_resources'))
regex = ".+(\.py|\.zip)$"
resource_listing = [f for f in resource_listing if re.match(regex, f)]
for f in resource_listing:
resource_local_path = os.path.join(local_base, "glue_py_resources", f)
path = upload_file_to_s3_from_path(resource_local_path, bucket, "{}/glue_py_resources/{}".format(bucket_folder,f))
# Remember to delete the files we downloaded
for this_path in delete_these_paths:
os.remove(this_path)