def train_and_save_gnosis_ref_arch_s3(training_data_url, fp_min_support_count,
fp_intent_topic_count_threshold,
fp_num_partition):
"""Train the Ref Arch Gnosis and saves the Gnosis model in S3.
:return: None
"""
input_bucket_name, output_bucket_name, additional_path = get_path_names(
training_data_url)
input_data_store = S3DataStore(src_bucket_name=input_bucket_name,
access_key=config.AWS_S3_ACCESS_KEY_ID,
secret_key=config.AWS_S3_SECRET_ACCESS_KEY)
output_data_store = S3DataStore(src_bucket_name=output_bucket_name,
access_key=config.AWS_S3_ACCESS_KEY_ID,
secret_key=config.AWS_S3_SECRET_ACCESS_KEY)
train_and_save_gnosis_ref_arch(
input_data_store=input_data_store,
output_data_store=output_data_store,
additional_path=additional_path,
fp_min_support_count=fp_min_support_count,
fp_intent_topic_count_threshold=fp_intent_topic_count_threshold,
fp_num_partition=fp_num_partition)
return None
def generate_evaluate_test_s3(training_url, result_id):
input_bucket_name, output_bucket_name, additional_path = get_path_names(
training_url)
input_data_store = S3DataStore(src_bucket_name=input_bucket_name,
access_key=AWS_S3_ACCESS_KEY_ID,
secret_key=AWS_S3_SECRET_ACCESS_KEY)
output_data_store = S3DataStore(src_bucket_name=input_bucket_name,
access_key=AWS_S3_ACCESS_KEY_ID,
secret_key=AWS_S3_SECRET_ACCESS_KEY)
generate_test(input_data_store, output_data_store, additional_path)
test_kronos(training_url, result_id, input_data_store, output_data_store,
additional_path)
def generate_and_save_package_frequency_dict_s3(training_data_url):
input_bucket_name, output_bucket_name, additional_path = get_path_names(
training_data_url)
input_data_store = S3DataStore(src_bucket_name=input_bucket_name,
access_key=config.AWS_S3_ACCESS_KEY_ID,
secret_key=config.AWS_S3_SECRET_ACCESS_KEY)
output_data_store = S3DataStore(src_bucket_name=output_bucket_name,
access_key=config.AWS_S3_ACCESS_KEY_ID,
secret_key=config.AWS_S3_SECRET_ACCESS_KEY)
frequency_dict_generator = FrequencyDictGenerator.create_frequency_generator(
input_data_store=input_data_store, additional_path=additional_path)
frequency_dict_generator.generate_and_save_frequency_dict(
output_data_store=output_data_store, additional_path=additional_path)
def generate_evaluate_test_s3(training_url, result_id):
"""Generate the test, save it, and then call all relevant checkers."""
input_bucket_name, output_bucket_name, additional_path = get_path_names(
training_url)
input_data_store = S3DataStore(src_bucket_name=input_bucket_name,
access_key=AWS_S3_ACCESS_KEY_ID,
secret_key=AWS_S3_SECRET_ACCESS_KEY)
output_data_store = S3DataStore(src_bucket_name=input_bucket_name,
access_key=AWS_S3_ACCESS_KEY_ID,
secret_key=AWS_S3_SECRET_ACCESS_KEY)
generate_test(input_data_store, output_data_store, additional_path)
perform_kronos_test(training_url, result_id, input_data_store,
output_data_store, additional_path)
def generate_and_save_gnosis_package_topic_model_s3(training_data_url):
"""Generate and save Gnosis package topic model into S3."""
input_bucket_name, output_bucket_name, additional_path = get_path_names(
training_data_url)
input_data_store = S3DataStore(src_bucket_name=input_bucket_name,
access_key=config.AWS_S3_ACCESS_KEY_ID,
secret_key=config.AWS_S3_SECRET_ACCESS_KEY)
output_data_store = S3DataStore(src_bucket_name=output_bucket_name,
access_key=config.AWS_S3_ACCESS_KEY_ID,
secret_key=config.AWS_S3_SECRET_ACCESS_KEY)
generate_and_save_gnosis_package_topic_model(input_data_store=input_data_store,
output_data_store=output_data_store,
additional_path=additional_path)
def train_and_save_pruned_tag_list_s3(training_data_url):
"""Return the clean package_topic present in the given s3 training URL.
:param training_data_url: The Location where data is read from and written to."""
input_bucket_name, output_bucket_name, additional_path = get_path_names(
training_data_url)
input_package_topic_data_store = S3DataStore(src_bucket_name=input_bucket_name,
access_key=config.AWS_S3_ACCESS_KEY_ID,
secret_key=config.AWS_S3_SECRET_ACCESS_KEY)
output_package_topic_data_store = S3DataStore(src_bucket_name=output_bucket_name,
access_key=config.AWS_S3_ACCESS_KEY_ID,
secret_key=config.AWS_S3_SECRET_ACCESS_KEY)
return TagListPruner.prune_tag_list(input_package_topic_data_store,
output_package_topic_data_store,
additional_path)
def generate_and_save_kronos_dependency_s3(training_data_url):
input_bucket_name, output_bucket_name, additional_path = get_path_names(
training_data_url)
input_gnosis_data_store = S3DataStore(src_bucket_name=input_bucket_name,
access_key=config.AWS_S3_ACCESS_KEY_ID,
secret_key=config.AWS_S3_SECRET_ACCESS_KEY)
input_package_topic_data_store = S3DataStore(src_bucket_name=input_bucket_name,
access_key=config.AWS_S3_ACCESS_KEY_ID,
secret_key=config.AWS_S3_SECRET_ACCESS_KEY)
output_data_store = S3DataStore(src_bucket_name=output_bucket_name,
access_key=config.AWS_S3_ACCESS_KEY_ID,
secret_key=config.AWS_S3_SECRET_ACCESS_KEY)
generate_and_save_kronos_dependency(
input_gnosis_data_store=input_gnosis_data_store,
input_package_topic_data_store=input_package_topic_data_store,
output_data_store=output_data_store, additional_path=additional_path)
def generate_and_save_cooccurrence_matrices_s3(training_data_url):
input_bucket_name, output_bucket_name, additional_path = get_path_names(
training_data_url)
input_kronos_dependency_data_store = S3DataStore(src_bucket_name=input_bucket_name,
access_key=config.AWS_S3_ACCESS_KEY_ID,
secret_key=config.AWS_S3_SECRET_ACCESS_KEY)
input_manifest_data_store = S3DataStore(src_bucket_name=input_bucket_name,
access_key=config.AWS_S3_ACCESS_KEY_ID,
secret_key=config.AWS_S3_SECRET_ACCESS_KEY)
output_data_store = S3DataStore(src_bucket_name=output_bucket_name,
access_key=config.AWS_S3_ACCESS_KEY_ID,
secret_key=config.AWS_S3_SECRET_ACCESS_KEY)
generate_and_save_cooccurrence_matrices(
input_kronos_dependency_data_store=input_kronos_dependency_data_store,
input_manifest_data_store=input_manifest_data_store,
output_data_store=output_data_store, additional_path=additional_path)
def train_and_save_kronos_list_s3(training_data_url):
"""Train the Kronos and save the results into the AWS S3 storage."""
input_bucket_name, output_bucket_name, additional_path = get_path_names(
training_data_url)
input_kronos_dependency_data_store = S3DataStore(src_bucket_name=input_bucket_name,
access_key=config.AWS_S3_ACCESS_KEY_ID,
secret_key=config.AWS_S3_SECRET_ACCESS_KEY)
input_cooccurrence_matrix_data_store = S3DataStore(src_bucket_name=input_bucket_name,
access_key=config.AWS_S3_ACCESS_KEY_ID,
secret_key=config.AWS_S3_SECRET_ACCESS_KEY)
output_data_store = S3DataStore(src_bucket_name=output_bucket_name,
access_key=config.AWS_S3_ACCESS_KEY_ID,
secret_key=config.AWS_S3_SECRET_ACCESS_KEY)
train_and_save_kronos_list(
input_kronos_dependency_data_store=input_kronos_dependency_data_store,
input_co_occurrence_data_store=input_cooccurrence_matrix_data_store,
output_data_store=output_data_store, additional_path=additional_path)