def getUpdateInstance_byFlask(self): print("Into ACR Update Instance by Flask") parameter = load_json_config("./parameter.json") list_args = parameter["acr_preprocess"] # acr_path = parameter["DATA_DIR"] + "/pickles/acr_articles_metadata_embeddings_predict/acr_articles_metadata_embeddings_predict.pickle" # acr_path = parameter["DATA_DIR"] + "/pickles/acr_articles_metadata_embeddings/acr_articles_metadata_embeddings.pickle" self.model_nar_dir = parameter["DATA_DIR"] + parameter["model_dir_nar"] (acr_label_encoders, articles_metadata_df, content_article_embeddings) = \ deserialize(self.acr_path) self.acr_label_encoders = acr_label_encoders self.articles_metadata_df = articles_metadata_df # self.reverse_acr_article_id = {} self.reverse_acr_article_id = { v: k for k, v in acr_label_encoders['article_id'].items() } self.content_article_embeddings_matrix = content_article_embeddings # get list_id list_id = get_list_id() encoded_list_id = [] for id in list_id: if (int(id) in acr_label_encoders['article_id']): encoded_list_id.append(self.get_article_id_encoded(int(id))) list_id_week_encoded = list(articles_metadata_df['article_id'])[-600:] encoded_list_id = list_id_week_encoded + encoded_list_id list_id_week = list(acr_label_encoders['article_id'].keys())[-600:] self.list_id = list(list_id) + list_id_week self.encoded_list_id = encoded_list_id # print("Loading ACR singleton") ACR_Pickle_Singleton.__instance = self print("Done Update SingleTon Flask Init Done")
def get_training_files(self, training_hour): parameter = load_json_config("./parameter.json") training_dir = parameter["DATA_DIR"] + parameter["nar_preprocess_2"][ "output_sessions_tfrecords_path"] train_files = resolve_files(training_dir)[-training_hour:] # print("TrainFile") # print(train_files) return list(chunks(train_files, training_hour))
def __init__(self): parameter = load_json_config("./parameter.json") list_args = parameter["acr_preprocess"] nar_path = parameter[ "DATA_DIR"] + "/pickles/nar_preprocessing_resources/nar_preprocessing_resources.pickle" self = load_nar_module_preprocessing_resources(nar_path) # print("Loading NAR singleton") NAR_Pickle_Singleton.__instance = self
def main_nar_preprocess_2(): #def main(): # parser = create_args_parser() # args = parser.parse_args() print("<=== STARTING NAR PREPROCESS 2 ===> ") # parameter = load_json_config("./parameter.json") parameter = load_json_config("./parameter.json") list_args = parameter["nar_preprocess_2"] DATA_DIR = parameter["DATA_DIR"] num_day = list_args["num_day"] input_sessions_json_folder_path = DATA_DIR + list_args[ "input_sessions_json_folder_path"] input_acr_metadata_embeddings_path = DATA_DIR + list_args[ "input_acr_metadata_embeddings_path"] input_nar_encoders_dict_path = DATA_DIR + list_args[ "input_nar_encoders_dict_path"] number_hours_to_preprocess = list_args["number_hours_to_preprocess"] output_nar_preprocessing_resources_path = DATA_DIR + list_args[ "output_nar_preprocessing_resources_path"] output_sessions_tfrecords_path = DATA_DIR + list_args[ "output_sessions_tfrecords_path"] if path.exists(output_nar_preprocessing_resources_path): pass else: import os os.makedirs(output_nar_preprocessing_resources_path) print('Loading resources generated ACR module (articles metadata)') # truyen file get_article_text_length_fn, get_article_id_encoded_fn = load_acr_module_resources( get_all_file(input_acr_metadata_embeddings_path)[0]) #get_article_text_length_fn = None # # degub # print(get_article_text_length_fn) print( 'Loading resources generated by the first step of NAR preprocessing (cat. features dict encoders)' ) nar_encoders_dict = load_nar_module_resources( get_all_file(input_nar_encoders_dict_path)[0]) print('Loading sessions from folder: {}'.format( input_sessions_json_folder_path)) print('Exporting TFRecords to: {}'.format(output_sessions_tfrecords_path)) # delete file .part* # from subprocess import Popen # var1 = DATA_DIR+input_sessions_json_folder_path+"session_hour=*/.*" # Process = Popen(['./nar_module/scripts/remove_hiden_file.sh %s' % str(var1)], shell=True) import os var1 = 'rm -rf ' + input_sessions_json_folder_path + "/session_hour=*/.*" print(var1) myCmd = var1 if os.system(myCmd) != 0: print("Xoa thanh cong") else: print("Xoa That bai") # split path output_sessions_tfrecords_path path_tf = DATA_DIR + '/' + list_args[ "output_sessions_tfrecords_path"].split('/')[1] if path.exists(path_tf): pass else: import os os.makedirs(path_tf) clicks_by_articles_counters = [] #a = preprocess_for_predict("2265891616712405988", get_article_text_length_fn, get_article_id_encoded_fn) for (hour_index, sessions_hour ) in load_sessions_hours(input_sessions_json_folder_path): # check directory empty: if len(os.listdir(DATA_DIR + "/sessions_tfrecords_by_hour/")) != 0: hour_index = split_string(DATA_DIR + "/sessions_tfrecords_by_hour/") + 1 print('Processing hour {}'.format(hour_index)) ####compute_global_metrics(sessions_hour) sessions_hour_df, hour_stats, hour_clicks_by_articles_counter = process_session_clicks_features( sessions_hour, get_article_text_length_fn) #sessions_hour_df.to_csv('hour-{}-to-debug.csv'.format(hour_index)) hour_stats['_hour_index'] = hour_index #stats.append(hour_stats) clicks_by_articles_counters.append(hour_clicks_by_articles_counter) # sessions_hour_df.to_csv(DATA_DIR+"/sessions_tfrecords_by_hour/sessions_hour_df.csv", index=False) export_sessions_hour_to_tf_records( hour_index, sessions_hour_df, output_path=output_sessions_tfrecords_path) # print('') # if number_hours_to_preprocess >= 0 and hour_index == number_hours_to_preprocess: # break print() print( 'Exporting Categorical Feature encoders and Numeric scalers dicts: {}'. format(output_nar_preprocessing_resources_path)) save_nar_preprocessing_resources( output_nar_preprocessing_resources_path + "nar_preprocessing_resources.pickle", nar_encoders_dict, numeric_scalers) # delete to keep tf record in 2 week nearest # after export tfrecord for trainning, delete all file in input_sessions_json_folder_path if len(os.listdir(DATA_DIR + "/sessions_tfrecords_by_hour/")) > 24 * num_day: delete_file_keep_in_two_week(DATA_DIR + "/sessions_tfrecords_by_hour/", 24 * num_day) # delete_all_file_in_path(input_sessions_json_folder_path) print("<=== END NAR PREPROCESS 2 ===> ")
def main_nar_preprocess_1(): spark = spark_inital() sc = spark.sparkContext sqlContext = SQLContext(sc) # get paramater parameter = load_json_config("./parameter.json") list_args = parameter["nar_preprocess_1"] DATA_DIR = parameter["DATA_DIR"] input_path_data_log_click = list_args["input_path_data_log_click"] date_start = list_args["date_start"] date_end = list_args["date_end"] input_path_proprcessed_cafebiz_articale_csv_from_acr = DATA_DIR + list_args[ "input_path_proprcessed_cafebiz_articale_csv_from_acr"] nar_encoders_cafebiz = DATA_DIR + list_args["nar_encoders_cafebiz"] input_articles_csv_path_original = DATA_DIR + list_args[ "input_articles_csv_path_original"] mysql_host = list_args["mysql_host"] mysql_user = list_args["mysql_user"] mysql_passwd = list_args["mysql_passwd"] mysql_database = list_args["mysql_database"] mysql_table = list_args["mysql_table"] domain = list_args["domain"] num_hour_trainning = list_args["n_hour_train_continue"] list_args2 = parameter["nar_preprocess_2"] spark_pre_json_path = DATA_DIR + list_args2[ 'input_sessions_json_folder_path'] session_tfrecord_path = DATA_DIR + list_args2[ 'output_sessions_tfrecords_path'] # Delete folder before run nar if path.exists(spark_pre_json_path): import shutil shutil.rmtree(spark_pre_json_path) # delete_all_file_in_path(split_string_train_path( session_tfrecord_path)) print("STARTNIG NAR PREPROCESSING ....") # database news print(input_articles_csv_path_original) df_news = handle_database_news_load_from_acr( input_articles_csv_path_original, spark) current_time = get_date_dt_data_log() flag = 0 if path.exists(DATA_DIR + "/sessions_tfrecords_by_hour/"): # second time # read date now , 1 hour before df_log = handle_database_log_mysql(num_hour_trainning, domain, spark, mysql_host, mysql_user, mysql_passwd, mysql_database, mysql_table, current_time) flag = 2 # run second times else: # first time # read log from data_start to date_end df_log = handle_database_log(domain, spark, input_path_data_log_click, date_start, date_end) flag = 1 # first time # join database news and database log df = df_log.join(df_news, 'full_url', 'inner').drop(df_news.domain) # print(df.printSchema()) print("<=== STARTING NAR PREPROCESSING 1 ===>") articles_original_df = load_cafebiz_article_from_acr( get_all_file(input_path_proprcessed_cafebiz_articale_csv_from_acr)[0]) # code runable valid_articles_urls_to_ids_dict = dict(articles_original_df[[ 'url', 'id_encoded' ]].apply(lambda x: (x['url'], x['id_encoded']), axis=1).values) valid_articles_urls_to_ids_dict_broadcast = spark.sparkContext.broadcast( valid_articles_urls_to_ids_dict) ### Loading user interactions ### TEST data_test # df = spark.read.parquet("file:///data1/ngocvb/Ngoc_COV/chameleon/nar_preprocess/nar_data/data_test/") # df = spark.read.parquet("file:///data/tungtv/jupytercode/data-log-news-parquet-thang45/") # df_news = handle_database_news_load_from_acr( # "/data/tungtv/Code/dataset/dataset_cafebiz_acr_nar_1/original_cafebiz_articles_csv/cafebiz_articles_original.csv", # spark) # df_log = handle_database_log(spark, "hdfs://10.5.37.76:8020/Data/Logging/pvTosFull/pc/", "2019-07-01", "2019-07-05") # df = df_log.join(df_news, 'full_url', 'inner').drop(df_news.domain) interactions_df = df.select("full_url", "dt", "os_code", "loc_id", "path" \ , "guid", "category0", "id", "content", "created_at_ts" \ , "teaser", "title", "keywords", "time", "url", "top") # .alias("full_url"), "id", "content", "created_at_ts", "teaser", "domain", "keywords", # "title", \ # "url", "category0", "persons", "locations", "text_highlights") # tructField("dt", StringType(), True) \ # , StructField("loc_id", IntegerType(), True) \ # , StructField("os_code", IntegerType(), True) \ # , StructField("top", IntegerType(), True) \ # , StructField("guid", StringType(), True) \ # , StructField("news_id", LongType(), True) \ # , StructField("domain", StringType(), True) \ # , StructField("path", StringType(), True)]) interactions_df = interactions_df.withColumn( "id", interactions_df["id"].cast(LongType())) interactions_df = interactions_df.withColumn( "created_at_ts", interactions_df["created_at_ts"].cast(LongType())) get_article_id_encoded_from_url_udf = F.udf( lambda url: get_article_id_encoded_from_url( url, valid_articles_urls_to_ids_dict_broadcast), pyspark.sql.types.IntegerType()) # Filtering only interactions whose url/id is available in the articles table # tungtv interactions_article_id_encoded_df = interactions_df.withColumn( 'article_id', get_article_id_encoded_from_url_udf(interactions_df['url'])) # interactions_article_id_encoded_df = interactions_df.withColumn('article_id', interactions_df['id']) interactions_filtered_df = interactions_article_id_encoded_df.filter( interactions_article_id_encoded_df['article_id'].isNull() == False) # print(interactions_filtered_df.printSchema()) # print(interactions_filtered_df.select("article_id")) first_timestamp_ts = interactions_filtered_df.select('time').agg( F.min('time')).collect()[0][0] * 1000 # Analyzing elapsed time since publishing interactions_filtered_df = add_column_dataframe( interactions_filtered_df, "publish_ts", F.to_timestamp( interactions_filtered_df.created_at_ts.cast( dataType=t.TimestampType()))) interactions_filtered_df = add_column_dataframe( interactions_filtered_df, "publish_ts", F.unix_timestamp(col("publish_ts"), 'yyyy-MM-dd HH:mm:ss')) get_timestamp_from_date_str_udf = F.udf(get_timestamp_from_date_str, pyspark.sql.types.IntegerType()) interactions_filtered_with_publish_ts_df = add_column_dataframe(interactions_filtered_df, "elapsed_min_since_published", \ ((F.col('time') - F.col('publish_ts')) / 60).cast( pyspark.sql.types.IntegerType())) interactions_filtered_with_publish_ts_df.approxQuantile( "elapsed_min_since_published", [0.10, 0.25, 0.50, 0.75, 0.90], 0.01) elapsed_min_since_published_df = interactions_filtered_with_publish_ts_df.select( 'elapsed_min_since_published').toPandas() """PAD_TOKEN = '<PAD>' UNFREQ_TOKEN = '<UNF>'""" # Analyzing clicks by article distribution ## Processing categorical features if flag == 1: cities_df = get_categ_features_counts_dataframe( interactions_filtered_df, 'loc_id') cities_encoder_dict = get_categ_features_encoder_dict(cities_df) os_df = get_categ_features_counts_dataframe(interactions_filtered_df, 'os_code') os_encoder_dict = get_categ_features_encoder_dict(os_df) else: # map value from nar encode city # read nar_encode_pickle nar_encoder_dict = deserialize(get_all_file(nar_encoders_cafebiz)[0]) cities_df = get_categ_features_counts_dataframe( interactions_filtered_df, 'loc_id') cities_encoder_dict = get_categ_features_encoder_dict_second_time( cities_df, nar_encoder_dict['os']) os_df = get_categ_features_counts_dataframe(interactions_filtered_df, 'os_code') os_encoder_dict = get_categ_features_encoder_dict_second_time( os_df, nar_encoder_dict['city']) encoders_dict = { 'city': cities_encoder_dict, # 'region': regions_encoder_dict, # 'country': countries_encoder_dict, 'os': os_encoder_dict, # 'device': devices_encoder_dict, # 'referrer_class': referrer_class_encoder_dict } # Processing numeric features active_time_quantiles = interactions_filtered_df.approxQuantile( "top", [0.10, 0.25, 0.50, 0.75, 0.90], 0.01) active_time_stats_df = interactions_filtered_df.describe('top').toPandas() active_time_mean = float(active_time_stats_df[ active_time_stats_df['summary'] == 'mean']['top'].values[0]) active_time_stddev = float(active_time_stats_df[ active_time_stats_df['summary'] == 'stddev']['top'].values[0]) interactions_filtered_df = interactions_filtered_df.withColumnRenamed( "guid", "user_id") interactions_filtered_df = interactions_filtered_df.orderBy("time") ### Splitting sessions MAX_SESSION_IDLE_TIME_MS = 30 * 60 * 1000 # 30 min # test_df = interactions_filtered_df.limit(1000).rdd.map(lambda x: (x['user_id'], x)) # print(test_df.take(10)) sessions_rdd = interactions_filtered_df.rdd.map(lambda x: (x['user_id'], x)).groupByKey() \ .flatMap(lambda row: split_sessions(row, encoders_dict, MAX_SESSION_IDLE_TIME_MS, first_timestamp_ts)) \ .sortByKey() \ .map(lambda x: x[1]) #### Exporting sessions to JSON lines sessions_sdf = sessions_rdd.toDF() sessions_sdf.write.partitionBy("session_hour").json( os.path.join(DATA_DIR, "sessions_processed_by_spark/")) if path.exists(nar_encoders_cafebiz): pass else: os.makedirs(nar_encoders_cafebiz) if flag == 1: # first time NAR_ENCODERS_PATH = 'nar_encoders_cafebiz.pickle' serialize(nar_encoders_cafebiz + NAR_ENCODERS_PATH, encoders_dict) print(" <=== END NAR PREPROCESSING 1 ===>")
def main(unused_argv): try: print("<=== STARTING NAR TRAINING ===> ") # load file parameter parameter = load_json_config("./parameter.json") data_dir = parameter['DATA_DIR'] list_args = parameter["nar_preprocess_2"] paht_session_spark = data_dir + list_args[ "input_sessions_json_folder_path"] + "/" # get num hour to train continue ... list_args_train = parameter["nar_preprocess_1"] hour_train_continue = list_args_train["n_hour_train_continue"] print(hour_train_continue) # Capture whether it will be a single training job or a hyper parameter tuning job. tf_config_env = json.loads(os.environ.get('TF_CONFIG', '{}')) task_data = tf_config_env.get('task') or {'type': 'master', 'index': 0} trial = task_data.get('trial') running_on_mlengine = (len(tf_config_env) > 0) print('Running {}'.format('on Google ML Engine' if running_on_mlengine else 'on a server/machine')) #Disabling duplicate logs on console when running locally logging.getLogger('tensorflow').propagate = running_on_mlengine tf.logging.info('Starting training job') gcs_model_output_dir = data_dir + FLAGS.model_dir # create folder model_dir if path.exists(gcs_model_output_dir): pass else: os.makedirs(gcs_model_output_dir) #If must persist and load model ouput in a local cache (to speedup in ML Engine) if FLAGS.use_local_cache_model_dir: model_output_dir = tempfile.mkdtemp() tf.logging.info( 'Created local temp folder for models output: {}'.format( model_output_dir)) else: model_output_dir = gcs_model_output_dir if trial is not None: model_output_dir = os.path.join(model_output_dir, trial) gcs_model_output_dir = os.path.join(gcs_model_output_dir, trial) tf.logging.info( "Hyperparameter Tuning - Trial {} - model_dir = {} - gcs_model_output_dir = {} " .format(trial, model_output_dir, gcs_model_output_dir)) tf.logging.info( 'Will save temporary model outputs to {}'.format(model_output_dir)) #If should warm start training from other previously trained model if FLAGS.warmup_model_dir != None: tf.logging.info( 'Copying model outputs from previous job ({}) for warm start'. format(FLAGS.warmup_model_dir)) dowload_model_output_from_gcs( model_output_dir, gcs_model_dir=FLAGS.warmup_model_dir, files_pattern=['graph.pb', 'model.ckpt-', 'checkpoint']) local_files_after_download_to_debug = list( glob.iglob("{}/**/*".format(model_output_dir), recursive=True)) tf.logging.info( 'Files copied from GCS to warm start training: {}'.format( local_files_after_download_to_debug)) print("STARTING ...") tf.logging.info('Loading ACR module assets') acr_label_encoders, articles_metadata_df, content_article_embeddings_matrix = \ load_acr_module_resources(get_all_file(data_dir + FLAGS.acr_module_resources_path)[0]) #Min-max scaling of the ACR embedding for a compatible range with other input features for NAR module # content_article_embeddings_matrix = min_max_scale(content_article_embeddings_matrix, min_max_range=(-0.1,0.1)) #Apply l2-norm by sample l2_normalizer_by_sample = Normalizer(norm='l2') content_article_embeddings_matrix = l2_normalizer_by_sample.fit_transform( content_article_embeddings_matrix) #Rescaling content features content_article_embeddings_matrix = content_article_embeddings_matrix * FLAGS.content_embedding_scale_factor articles_features_config = get_articles_features_config( acr_label_encoders) articles_metadata = process_articles_metadata( articles_metadata_df, articles_features_config) tf.logging.info('Loading NAR module preprocesing assets') nar_label_encoders = load_nar_module_preprocessing_resources( get_all_file(data_dir + FLAGS.nar_module_preprocessing_resources_path)[0]) session_features_config = get_session_features_config( nar_label_encoders) tf.logging.info('Building NAR model') global eval_sessions_metrics_log, clicked_items_state, sessions_negative_items_log, sessions_chameleon_recommendations_log, global_eval_hour_id eval_sessions_metrics_log = [] clicked_items_state = ClickedItemsState( FLAGS.recent_clicks_buffer_hours, FLAGS.recent_clicks_buffer_max_size, FLAGS.recent_clicks_for_normalization, content_article_embeddings_matrix.shape[0]) tf.logging.info('Getting training file names') train_files = resolve_files(data_dir + FLAGS.train_set_path_regex) # get number of file in trani folder, to find train_file_from, train_file_to train_files_from = 0 train_files_up_to = get_train_file_max( split_string_train_path(data_dir + FLAGS.train_set_path_regex)) # train_files_up_to = 15 if train_files_from > train_files_up_to: raise Exception( 'Final training file cannot be lower than Starting training file' ) train_files = train_files[train_files_from:train_files_up_to] print("============================>") print("Len file train: {}".format(len(train_files))) tf.logging.info('{} files where the network will be trained and evaluated on, from {} to {}' \ .format(len(train_files), train_files[0], train_files[-1])) if len(train_files) <= hour_train_continue: print("run_config for second time") run_config = tf.estimator.RunConfig( tf_random_seed=RANDOM_SEED, keep_checkpoint_max=1, # save_checkpoints_secs=1200, save_checkpoints_steps=1, save_summary_steps=100, log_step_count_steps=100, # session_config=config_proto ) else: print("run_config for frist time") run_config = tf.estimator.RunConfig( tf_random_seed=RANDOM_SEED, keep_checkpoint_max=1, save_checkpoints_secs=1200, # save_checkpoints_steps=1, save_summary_steps=100, log_step_count_steps=100, # session_config=config_proto ) model = build_estimator(run_config, model_output_dir, content_article_embeddings_matrix, articles_metadata, articles_features_config, session_features_config) start_train = time() tf.logging.info("Starting Training Loop") training_files_chunks = list( chunks(train_files, FLAGS.training_hours_for_each_eval)) for chunk_id in range(0, len(training_files_chunks)): training_files_chunk = training_files_chunks[chunk_id] tf.logging.info('Training files from {} to {}'.format( training_files_chunk[0], training_files_chunk[-1])) model.train(input_fn=lambda: prepare_dataset_iterator( training_files_chunk, session_features_config, batch_size=FLAGS.batch_size, truncate_session_length=FLAGS.truncate_session_length)) if chunk_id < len(training_files_chunks) - 1: #Using the first hour of next training chunck as eval eval_file = training_files_chunks[chunk_id + 1][0] tf.logging.info('Evaluating file {}'.format(eval_file)) model.evaluate(input_fn=lambda: prepare_dataset_iterator( eval_file, session_features_config, batch_size=FLAGS.batch_size, truncate_session_length=FLAGS.truncate_session_length)) #After each number of train/eval loops if chunk_id % FLAGS.save_results_each_n_evals == 0: tf.logging.info('Saving eval metrics') global_eval_hour_id += 1 save_eval_benchmark_metrics_csv( eval_sessions_metrics_log, model_output_dir, training_hours_for_each_eval=FLAGS. training_hours_for_each_eval) if FLAGS.save_eval_sessions_negative_samples: #Flushing to disk the negative samples used to evaluate each sessions, #so that benchmarks metrics outside the framework (eg. Matrix Factorization) can be comparable save_sessions_negative_items(model_output_dir, sessions_negative_items_log) sessions_negative_items_log = [] if FLAGS.save_eval_sessions_recommendations: #Flushing to disk the recommended items to test re-ranking approaches (e.g. MMR) save_sessions_chameleon_recommendations_log( model_output_dir, sessions_chameleon_recommendations_log, global_eval_hour_id) sessions_chameleon_recommendations_log = [] #Incrementing the eval hour id global_eval_hour_id += 1 #If must persist and load model ouput in a local cache (to speedup in ML Engine) if FLAGS.use_local_cache_model_dir: tf.logging.info('Uploading cached results to GCS') upload_model_output_to_gcs( model_output_dir, gcs_model_dir=gcs_model_output_dir, #files_pattern=None) files_pattern=[ #'events.out.tfevents.', '.csv', '.json' ]) tf.logging.info('Finalized Training') save_eval_benchmark_metrics_csv( eval_sessions_metrics_log, model_output_dir, training_hours_for_each_eval=FLAGS.training_hours_for_each_eval) if FLAGS.save_eval_sessions_negative_samples: #Flushing to disk the negative samples used to evaluate each sessions, #so that benchmarks metrics outside the framework (eg. Matrix Factorization) can be comparable save_sessions_negative_items(model_output_dir, sessions_negative_items_log) if FLAGS.save_eval_sessions_recommendations: #Flushing to disk the recommended items to test re-ranking approaches (e.g. MMR) save_sessions_chameleon_recommendations_log( model_output_dir, sessions_chameleon_recommendations_log, global_eval_hour_id) tf.logging.info('Saved eval metrics') #If must persist and load model ouput in a local cache (to speedup in ML Engine) if FLAGS.use_local_cache_model_dir: #Uploads all files to GCS upload_model_output_to_gcs(model_output_dir, gcs_model_dir=gcs_model_output_dir, files_pattern=None) # after training , delete all file in train file path, and delete dir sessions_processed_by_spark delete_all_file_in_path( split_string_train_path(data_dir + FLAGS.train_set_path_regex)) import shutil shutil.rmtree(paht_session_spark) # os.rmdir(paht_session_spark) # delete all file event in model_nar for filename in glob.glob(model_output_dir + "/events.out.tfevents*"): os.remove(filename) log_elapsed_time(start_train, 'Finalized TRAINING Loop') print("<=== END NAR TRAINING ===> ") except Exception as ex: tf.logging.error('ERROR: {}'.format(ex)) raise
def main_acr_train(): # def main(unused_argv): # try: print("<=== STARTING ARC TRAINING ===>") parameter = load_json_config("./parameter.json") list_args = parameter["acr_training"] DATA_DIR = parameter["DATA_DIR"] model_dir = DATA_DIR + list_args["model_dir"] train_set_path_regex = DATA_DIR + list_args["train_set_path_regex"] input_word_vocab_embeddings_path = DATA_DIR + list_args[ "input_word_vocab_embeddings_path"] input_label_encoders_path = DATA_DIR + list_args[ "input_label_encoders_path"] output_acr_metadata_embeddings_path = DATA_DIR + list_args[ "output_acr_metadata_embeddings_path"] batch_size = list_args["batch_size"] truncate_tokens_length = list_args["truncate_tokens_length"] training_epochs = list_args["training_epochs"] learning_rate = list_args["learning_rate"] dropout_keep_prob = list_args["dropout_keep_prob"] l2_reg_lambda = list_args["l2_reg_lambda"] text_feature_extractor = list_args["text_feature_extractor"] training_task = list_args["training_task"] cnn_filter_sizes = list_args["cnn_filter_sizes"] cnn_num_filters = list_args["cnn_num_filters"] rnn_units = list_args["rnn_units"] rnn_layers = list_args["rnn_layers"] rnn_direction = list_args["rnn_direction"] acr_embeddings_size = list_args["acr_embeddings_size"] # mysql_host = list_args["mysql_host"] # mysql_user = list_args["mysql_user"] # mysql_passwd = list_args["mysql_passwd"] # mysql_database = list_args["mysql_database"] # Capture whether it will be a single training job or a hyper parameter tuning job. tf_config_env = json.loads(os.environ.get('TF_CONFIG', '{}')) task_data = tf_config_env.get('task') or {'type': 'master', 'index': 0} trial = task_data.get('trial') running_on_mlengine = (len(tf_config_env) > 0) tf.logging.info( 'Running {}'.format('on Google ML Engine' if running_on_mlengine else 'on a server/machine')) #Disabling duplicate logs on console when running locally logging.getLogger('tensorflow').propagate = running_on_mlengine start_train = time() tf.logging.info('Starting training job') model_output_dir = model_dir if trial is not None: model_output_dir = os.path.join(model_output_dir, trial) tf.logging.info( "Hyperparameter Tuning - Trial {}. model_dir = {}".format( trial, model_output_dir)) else: tf.logging.info('Saving model outputs to {}'.format(model_output_dir)) tf.logging.info('Loading ACR preprocessing assets') # check exist path if path.exists(model_output_dir): pass else: os.makedirs(model_output_dir) if path.exists(output_acr_metadata_embeddings_path): pass else: os.makedirs(output_acr_metadata_embeddings_path) print("Loading ACR preprocessing assets....") print(input_label_encoders_path) print(output_acr_metadata_embeddings_path) file_lable_encode = get_all_file(input_label_encoders_path)[0] file_word_embedding = get_all_file(input_word_vocab_embeddings_path)[0] # current_time = split_date(file_lable_encode) # print(current_time) # load file with max date acr_label_encoders, labels_class_weights, word_embeddings_matrix = \ load_acr_preprocessing_assets(file_lable_encode,file_word_embedding) features_config = get_session_features_config(acr_label_encoders) #input_tfrecords = os.path.join(FLAGS.data_dir, FLAGS.train_set_path_regex) input_tfrecords = train_set_path_regex tf.logging.info( 'Defining input data (TFRecords): {}'.format(input_tfrecords)) #Creating an ambedding for a special token to initiate decoding of RNN-autoencoder special_token_embedding_vector = np.random.uniform( low=-0.04, high=0.04, size=[1, word_embeddings_matrix.shape[1]]) # train_files = get_listmax_date(get_all_file(train_set_path_regex)) train_files = get_all_file(train_set_path_regex) print(train_files) if len(os.listdir(model_dir)) == 0: #acr_model not exist print("NO Have ACR Module") acr_model = build_acr_estimator(model_output_dir, word_embeddings_matrix, features_config, labels_class_weights, special_token_embedding_vector, list_args) tf.logging.info('Training model') acr_model.train( input_fn=lambda: prepare_dataset(files=train_files, features_config=features_config, batch_size=batch_size, epochs=training_epochs, shuffle_dataset=True, shuffle_buffer_size=10000)) else: #acr_model exist print("Have ACR Module") acr_model = build_acr_estimator(model_output_dir, word_embeddings_matrix, features_config, labels_class_weights, special_token_embedding_vector, list_args) #The objective is to overfitting this network, so that the ACR embedding represent well the articles content tf.logging.info('Evaluating model - TRAIN SET') print("Evaluating model - TRAIN SET") eval_results = acr_model.evaluate( input_fn=lambda: prepare_dataset(files=train_files, features_config=features_config, batch_size=batch_size, epochs=1, shuffle_dataset=False)) tf.logging.info( 'Evaluation results with TRAIN SET (objective is to overfit): {}'. format(eval_results)) ''' tf.logging.info('Evaluating model - TEST SET') eval_results = acr_model.evaluate(input_fn=lambda: prepare_dataset(files=test_files, features_config=features_config, batch_size=FLAGS.batch_size, epochs=1, shuffle_dataset=False)) tf.logging.info('Evaluation results with TEST SET: {}'.format(eval_results)) ''' tf.logging.info('Predicting ACR embeddings') print("Predicting ACR embeddings") article_metadata_with_pred_embeddings = acr_model.predict( input_fn=lambda: prepare_dataset(files=train_files, features_config=features_config, batch_size=batch_size, epochs=1, shuffle_dataset=False)) articles_metadata_df, content_article_embeddings = get_articles_metadata_embeddings( article_metadata_with_pred_embeddings) tf.logging.info('Generated ACR embeddings: {}'.format( content_article_embeddings.shape)) # read csv preprocessed by acr preprocessing list_args2 = parameter["acr_preprocess"] path_csv = DATA_DIR + list_args2['output_articles_csv_path_preprocessed'] df = pd.read_csv(get_all_file(path_csv)[0]) print(len(df['id'])) export_acr_metadata_embeddings_with_datetime( df, acr_label_encoders, articles_metadata_df, content_article_embeddings, output_acr_metadata_embeddings_path) print("Export done, Call load acr auto ...") print("Remove acr embedding") remove_acr_pickle(get_all_file(output_acr_metadata_embeddings_path)[0]) # TODO gọi service load acr_label_encoders, articles_metadata_df, content_article_embeddings vào biến singleton # import requests # resp = requests.get('http://0.0.0.0:8082/loadacr') # if resp.status_code == 200: # print("Called load acr_pickle") # else: # print("Not Yet call load acr_pickle") # save_to_mysql_database( mysql_host, mysql_user, mysql_passwd, mysql_database, acr_label_encoders,articles_metadata_df , content_article_embeddings) # after trainning, delete all file tfrecord delete_all_file_in_path(train_set_path_regex) log_elapsed_time(start_train, 'Finalized TRAINING') print("<=== END ARC TRAINING ===>")
a = dt_object.strftime('%Y-%m-%d') import datetime import dateutil.relativedelta d = datetime.datetime.strptime(a, "%Y-%m-%d") d2 = d - dateutil.relativedelta.relativedelta(days=7) # print(d2) from datetime import datetime timestamp = datetime.timestamp(d2) return int(timestamp) # REMOVE PICKLE parameter = load_json_config("./parameter.json") def remove_acr_pickle(path_file): acr_label_encoders, articles_metadata_df, content_article_embeddings_matrix = deserialize( path_file) def serialize(filename, obj): # with open(filename, 'wb') as handle: with tf.gfile.Open(filename, 'wb') as handle: pickle.dump(obj, handle) def merge_two_dicts(x, y): return {**x, **y} # articles_metadata_df = articles_metadata_df[