def run_job(input_data_path=''):
if not input_data_path:
logger.warning("No input data path given, gracefully exiting.")
return
input_bucket_name, key = input_data_path.split('/', 2)[-1].split('/', 1)
input_bucket = S3DataStore(src_bucket_name=input_bucket_name,
access_key=config.AWS_S3_ACCESS_KEY_ID,
secret_key=config.AWS_S3_SECRET_ACCESS_KEY)
core_data = S3DataStore(src_bucket_name='prod-bayesian-core-data',
access_key=config.AWS_S3_ACCESS_KEY_ID,
secret_key=config.AWS_S3_SECRET_ACCESS_KEY)
packages = input_bucket.read_json_file(key)
count = 1
version_info = {}
for package in packages:
versions = sorted(core_data.list_folders(prefix=os.path.join('npm', package)), reverse=True)
if not versions:
logger.warning("[MISSING_DATA] Do not have data for any "
"versions of package {}".format(package))
continue
try:
version = versions[0].split('/')[2]
except Exception:
logger.warning("[KEY_FORMAT] Could not get version for {}".format(package))
version_info[package] = version
print(count)
count += 1
input_bucket.write_json_file('tagging/npm/missing_data/missing_version_info.json', version_info)
def generate_evaluate_test_s3(training_url, result_id):
input_bucket_name, output_bucket_name, additional_path = get_path_names(
training_url)
input_data_store = S3DataStore(src_bucket_name=input_bucket_name,
access_key=AWS_S3_ACCESS_KEY_ID,
secret_key=AWS_S3_SECRET_ACCESS_KEY)
output_data_store = S3DataStore(src_bucket_name=input_bucket_name,
access_key=AWS_S3_ACCESS_KEY_ID,
secret_key=AWS_S3_SECRET_ACCESS_KEY)
generate_test(input_data_store, output_data_store, additional_path)
test_kronos(training_url, result_id, input_data_store, output_data_store,
additional_path)
def generate_evaluate_test_s3(training_url, result_id):
"""Generate the test, save it, and then call all relevant checkers."""
input_bucket_name, output_bucket_name, additional_path = get_path_names(
training_url)
input_data_store = S3DataStore(src_bucket_name=input_bucket_name,
access_key=AWS_S3_ACCESS_KEY_ID,
secret_key=AWS_S3_SECRET_ACCESS_KEY)
output_data_store = S3DataStore(src_bucket_name=input_bucket_name,
access_key=AWS_S3_ACCESS_KEY_ID,
secret_key=AWS_S3_SECRET_ACCESS_KEY)
generate_test(input_data_store, output_data_store, additional_path)
perform_kronos_test(training_url, result_id, input_data_store,
output_data_store, additional_path)
def generate_and_save_gnosis_package_topic_model_s3(training_data_url):
"""Generate and save Gnosis package topic model into S3."""
input_bucket_name, output_bucket_name, additional_path = get_path_names(
training_data_url)
input_data_store = S3DataStore(src_bucket_name=input_bucket_name,
access_key=config.AWS_S3_ACCESS_KEY_ID,
secret_key=config.AWS_S3_SECRET_ACCESS_KEY)
output_data_store = S3DataStore(src_bucket_name=output_bucket_name,
access_key=config.AWS_S3_ACCESS_KEY_ID,
secret_key=config.AWS_S3_SECRET_ACCESS_KEY)
generate_and_save_gnosis_package_topic_model(input_data_store=input_data_store,
output_data_store=output_data_store,
additional_path=additional_path)
def train_and_save_pruned_tag_list_s3(training_data_url):
"""Return the clean package_topic present in the given s3 training URL.
:param training_data_url: The Location where data is read from and written to."""
input_bucket_name, output_bucket_name, additional_path = get_path_names(
training_data_url)
input_package_topic_data_store = S3DataStore(src_bucket_name=input_bucket_name,
access_key=config.AWS_S3_ACCESS_KEY_ID,
secret_key=config.AWS_S3_SECRET_ACCESS_KEY)
output_package_topic_data_store = S3DataStore(src_bucket_name=output_bucket_name,
access_key=config.AWS_S3_ACCESS_KEY_ID,
secret_key=config.AWS_S3_SECRET_ACCESS_KEY)
return TagListPruner.prune_tag_list(input_package_topic_data_store,
output_package_topic_data_store,
additional_path)
def generate_and_save_gnosis_package_topic_model_s3(training_data_url):
input_bucket_name = trunc_string_at(training_data_url, "/", 2, 3)
output_bucket_name = trunc_string_at(training_data_url, "/", 2, 3)
additional_path = trunc_string_at(training_data_url, "/", 3, -1)
input_data_store = S3DataStore(src_bucket_name=input_bucket_name,
access_key=config.AWS_S3_ACCESS_KEY_ID,
secret_key=config.AWS_S3_SECRET_ACCESS_KEY)
output_data_store = S3DataStore(src_bucket_name=output_bucket_name,
access_key=config.AWS_S3_ACCESS_KEY_ID,
secret_key=config.AWS_S3_SECRET_ACCESS_KEY)
generate_and_save_gnosis_package_topic_model(
input_data_store=input_data_store,
output_data_store=output_data_store,
additional_path=additional_path)
def load_package_frequency_dict_s3(bucket_name, additional_path):
input_data_store = S3DataStore(src_bucket_name=bucket_name,
access_key=config.AWS_S3_ACCESS_KEY_ID,
secret_key=config.AWS_S3_SECRET_ACCESS_KEY)
package_frequency_dict = load_package_frequency_dict(
input_data_store, additional_path)
return package_frequency_dict
def load_user_eco_to_kronos_model_dict_s3(bucket_name, additional_path):
input_data_store = S3DataStore(src_bucket_name=bucket_name,
access_key=config.AWS_S3_ACCESS_KEY_ID,
secret_key=config.AWS_S3_SECRET_ACCESS_KEY)
user_eco_to_kronos_model_dict = load_user_eco_to_kronos_model_dict(
input_kronos_data_store=input_data_store,
additional_path=additional_path)
return user_eco_to_kronos_model_dict
def load_eco_to_kronos_dependency_dict_s3(bucket_name, additional_path):
"""Load the Kronos dependency dictionary from the AWS S3 storage."""
input_data_store = S3DataStore(src_bucket_name=bucket_name,
access_key=config.AWS_S3_ACCESS_KEY_ID,
secret_key=config.AWS_S3_SECRET_ACCESS_KEY)
eco_to_kronos_dependency_dict = load_eco_to_kronos_dependency_dict(
input_kronos_dependency_data_store=input_data_store, additional_path=additional_path)
return eco_to_kronos_dependency_dict
def generate_and_save_kronos_dependency_s3(training_data_url):
input_bucket_name, output_bucket_name, additional_path = get_path_names(
training_data_url)
input_gnosis_data_store = S3DataStore(src_bucket_name=input_bucket_name,
access_key=config.AWS_S3_ACCESS_KEY_ID,
secret_key=config.AWS_S3_SECRET_ACCESS_KEY)
input_package_topic_data_store = S3DataStore(src_bucket_name=input_bucket_name,
access_key=config.AWS_S3_ACCESS_KEY_ID,
secret_key=config.AWS_S3_SECRET_ACCESS_KEY)
output_data_store = S3DataStore(src_bucket_name=output_bucket_name,
access_key=config.AWS_S3_ACCESS_KEY_ID,
secret_key=config.AWS_S3_SECRET_ACCESS_KEY)
generate_and_save_kronos_dependency(
input_gnosis_data_store=input_gnosis_data_store,
input_package_topic_data_store=input_package_topic_data_store,
output_data_store=output_data_store, additional_path=additional_path)
def generate_and_save_cooccurrence_matrices_s3(training_data_url):
input_bucket_name, output_bucket_name, additional_path = get_path_names(
training_data_url)
input_kronos_dependency_data_store = S3DataStore(src_bucket_name=input_bucket_name,
access_key=config.AWS_S3_ACCESS_KEY_ID,
secret_key=config.AWS_S3_SECRET_ACCESS_KEY)
input_manifest_data_store = S3DataStore(src_bucket_name=input_bucket_name,
access_key=config.AWS_S3_ACCESS_KEY_ID,
secret_key=config.AWS_S3_SECRET_ACCESS_KEY)
output_data_store = S3DataStore(src_bucket_name=output_bucket_name,
access_key=config.AWS_S3_ACCESS_KEY_ID,
secret_key=config.AWS_S3_SECRET_ACCESS_KEY)
generate_and_save_cooccurrence_matrices(
input_kronos_dependency_data_store=input_kronos_dependency_data_store,
input_manifest_data_store=input_manifest_data_store,
output_data_store=output_data_store, additional_path=additional_path)
def train_and_save_kronos_list_s3(training_data_url):
"""Train the Kronos and save the results into the AWS S3 storage."""
input_bucket_name, output_bucket_name, additional_path = get_path_names(
training_data_url)
input_kronos_dependency_data_store = S3DataStore(src_bucket_name=input_bucket_name,
access_key=config.AWS_S3_ACCESS_KEY_ID,
secret_key=config.AWS_S3_SECRET_ACCESS_KEY)
input_cooccurrence_matrix_data_store = S3DataStore(src_bucket_name=input_bucket_name,
access_key=config.AWS_S3_ACCESS_KEY_ID,
secret_key=config.AWS_S3_SECRET_ACCESS_KEY)
output_data_store = S3DataStore(src_bucket_name=output_bucket_name,
access_key=config.AWS_S3_ACCESS_KEY_ID,
secret_key=config.AWS_S3_SECRET_ACCESS_KEY)
train_and_save_kronos_list(
input_kronos_dependency_data_store=input_kronos_dependency_data_store,
input_co_occurrence_data_store=input_cooccurrence_matrix_data_store,
output_data_store=output_data_store, additional_path=additional_path)
def train_and_save_kronos_list_s3(training_data_url):
input_bucket_name = trunc_string_at(training_data_url, "/", 2, 3)
output_bucket_name = trunc_string_at(training_data_url, "/", 2, 3)
additional_path = trunc_string_at(training_data_url, "/", 3, -1)
input_kronos_dependency_data_store = S3DataStore(src_bucket_name=input_bucket_name,
access_key=config.AWS_S3_ACCESS_KEY_ID,
secret_key=config.AWS_S3_SECRET_ACCESS_KEY)
input_cooccurrence_matrix_data_store = S3DataStore(src_bucket_name=input_bucket_name,
access_key=config.AWS_S3_ACCESS_KEY_ID,
secret_key=config.AWS_S3_SECRET_ACCESS_KEY)
output_data_store = S3DataStore(src_bucket_name=output_bucket_name,
access_key=config.AWS_S3_ACCESS_KEY_ID,
secret_key=config.AWS_S3_SECRET_ACCESS_KEY)
train_and_save_kronos_list(
input_kronos_dependency_data_store=input_kronos_dependency_data_store,
input_co_occurrence_data_store=input_cooccurrence_matrix_data_store,
output_data_store=output_data_store, additional_path=additional_path)
def load_model():
print "Loading model ..."
bucket_name = config.AWS_BUCKET
access_key_id = config.AWS_ACCESS_KEY_ID
secret_access_key = config.AWS_SECRET_ACCESS_KEY
print "Read the config values ..."
print "------------"
print access_key_id
print "------------"
model_data_store = S3DataStore(src_bucket_name=bucket_name.strip(),
access_key=access_key_id.strip(),
secret_key=secret_access_key.strip())
assert (model_data_store is not None)
app.movie_reco_model = MovieRecommender.load_from_data_store(data_store=model_data_store)
assert app.movie_reco_model is not None
app.logger.info("movie recommendation model got loaded successfully!")
def load_rec_model_s3():
data_store = S3DataStore(src_bucket_name=AWS_BUCKET_NAME,
access_key=AWS_S3_ACCESS_KEY_ID,
secret_key=AWS_S3_SECRET_ACCESS_KEY)
rec_sys = ImdbRecSys.load(data_store=data_store)
return rec_sys