def add_dataset_to_project(project_id, file_name): """Add file path to the project file. Add file to data subfolder and fill the pool of iteration 0. """ project_file_path = get_project_file_path(project_id) fp_lock = get_lock_path(project_id) with SQLiteLock(fp_lock, blocking=True, lock_name="active"): # open the projects file with open(project_file_path, "r") as f_read: project_dict = json.load(f_read) # add path to dict (overwrite if already exists) project_dict["dataset_path"] = file_name with open(project_file_path, "w") as f_write: json.dump(project_dict, f_write) # fill the pool of the first iteration pool_indices = read_data(project_id).record_ids np.random.shuffle(pool_indices) write_pool(project_id, pool_indices.tolist()) # make a empty qeue for the items to label write_label_history(project_id, [])
def move_label_from_labeled_to_pool(project_id, paper_i): # load the papers from the pool pool_list = read_pool(project_id) # Add the paper to the reviewed papers. labeled_list = read_label_history(project_id) labeled_list_new = [] for item_id, item_label in labeled_list: item_id = int(item_id) item_label = int(item_label) paper_i = int(paper_i) if paper_i == item_id: pool_list.append(item_id) else: labeled_list_new.append([item_id, item_label]) # write the papers to the label dataset write_pool(project_id, pool_list) # load the papers from the pool write_label_history(project_id, labeled_list_new)
def add_dataset_to_project(project_id, file_name): """Add file path to the project file. Add file to data subfolder and fill the pool of iteration 0. """ project_file_path = get_project_file_path(project_id) # clean temp project files clean_project_tmp_files(project_id) with SQLiteLock( get_lock_path(project_id), blocking=True, lock_name="active", project_id=project_id ): # open the projects file with open(project_file_path, "r") as f_read: project_dict = json.load(f_read) # add path to dict (overwrite if already exists) project_dict["dataset_path"] = file_name with open(project_file_path, "w") as f_write: json.dump(project_dict, f_write) # fill the pool of the first iteration as_data = read_data(project_id) if as_data.labels is not None: unlabeled = np.where(as_data.labels == LABEL_NA)[0] pool_indices = as_data.record_ids[unlabeled] labeled_indices = np.where(as_data.labels != LABEL_NA)[0] label_indices = list(zip( as_data.record_ids[labeled_indices].tolist(), as_data.labels[labeled_indices].tolist() )) else: pool_indices = as_data.record_ids label_indices = [] np.random.shuffle(pool_indices) write_pool(project_id, pool_indices.tolist()) # make a empty qeue for the items to label write_label_history(project_id, label_indices)
def move_label_from_pool_to_labeled(project_id, paper_i, label): # load the papers from the pool pool_idx = read_pool(project_id) # Remove the paper from the pool. try: pool_idx.remove(int(paper_i)) except (IndexError, ValueError): return write_pool(project_id, pool_idx) # Add the paper to the reviewed papers. labeled = read_label_history(project_id) labeled.append([int(paper_i), int(label)]) write_label_history(project_id, labeled)
def add_dataset_to_project(project_id, file_name): """Add file path to the project file. Add file to data subfolder and fill the pool of iteration 0. """ project_file_path = get_project_file_path(project_id) fp_lock = get_lock_path(project_id) with SQLiteLock(fp_lock, blocking=True, lock_name="active"): # open the projects file with open(project_file_path, "r") as f_read: project_dict = json.load(f_read) # add path to dict (overwrite if already exists) project_dict["dataset_path"] = file_name with open(project_file_path, "w") as f_write: json.dump(project_dict, f_write) # fill the pool of the first iteration as_data = read_data(project_id) if as_data.labels is not None: unlabeled = np.where(as_data.labels == LABEL_NA)[0] pool_indices = as_data.record_ids[unlabeled] label_indices_included = \ [[int(x), 1] for x in np.where(as_data.labels == 1)[0]] label_indices_excluded = \ [[int(x), 0] for x in np.where(as_data.labels == 0)[0]] label_indices = label_indices_included + label_indices_excluded else: pool_indices = as_data.record_ids label_indices = [] np.random.shuffle(pool_indices) write_pool(project_id, pool_indices.tolist()) # make a empty qeue for the items to label write_label_history(project_id, label_indices)
def train_model(project_id, label_method=None): """Add the new labels to the review and do the modeling. It uses a lock to ensure only one model is running at the same time. Old results directories are deleted after 4 iterations. It has one argument on the CLI, which is the base project directory. """ logging.info(f"Project {project_id} - Train a new model for project") # get file locations asr_kwargs_file = get_kwargs_path(project_id) lock_file = get_lock_path(project_id) # Lock so that only one training run is running at the same time. # It doesn't lock the flask server/client. with SQLiteLock(lock_file, blocking=False, lock_name="training", project_id=project_id) as lock: # If the lock is not acquired, another training instance is running. if not lock.locked(): logging.info("Project {project_id} - " "Cannot acquire lock, other instance running.") return # Lock the current state. We want to have a consistent active state. # This does communicate with the flask backend; it prevents writing and # reading to the same files at the same time. with SQLiteLock(lock_file, blocking=True, lock_name="active", project_id=project_id) as lock: # Get the all labels since last run. If no new labels, quit. new_label_history = read_label_history(project_id) data_fp = str(get_data_file_path(project_id)) as_data = read_data(project_id) state_file = get_state_path(project_id) # collect command line arguments and pass them to the reviewer with open(asr_kwargs_file, "r") as fp: asr_kwargs = json.load(fp) asr_kwargs['state_file'] = str(state_file) reviewer = get_reviewer(dataset=data_fp, mode="minimal", **asr_kwargs) with open_state(state_file) as state: old_label_history = get_label_train_history(state) diff_history = get_diff_history(new_label_history, old_label_history) if len(diff_history) == 0: logging.info( "Project {project_id} - No new labels since last run.") return query_idx = np.array([x[0] for x in diff_history], dtype=int) inclusions = np.array([x[1] for x in diff_history], dtype=int) # Classify the new labels, train and store the results. with open_state(state_file) as state: reviewer.classify(query_idx, inclusions, state, method=label_method) reviewer.train() reviewer.log_probabilities(state) new_query_idx = reviewer.query(reviewer.n_pool()).tolist() reviewer.log_current_query(state) proba = state.pred_proba.tolist() with SQLiteLock(lock_file, blocking=True, lock_name="active", project_id=project_id) as lock: current_pool = read_pool(project_id) in_current_pool = np.zeros(len(as_data)) in_current_pool[current_pool] = 1 new_pool = [x for x in new_query_idx if in_current_pool[x]] write_pool(project_id, new_pool) write_proba(project_id, proba)
def train_model(project_id, label_method=None): """Add the new labels to the review and do the modeling. It uses a lock to ensure only one model is running at the same time. Old results directories are deleted after 4 iterations. It has one argument on the CLI, which is the base project directory. """ logging.info(f"Project {project_id} - Train a new model for project") # get file locations asr_kwargs_file = get_kwargs_path(project_id) lock_file = get_lock_path(project_id) # Lock so that only one training run is running at the same time. # It doesn't lock the flask server/client. with SQLiteLock( lock_file, blocking=False, lock_name="training", project_id=project_id) as lock: # If the lock is not acquired, another training instance is running. if not lock.locked(): logging.info("Project {project_id} - " "Cannot acquire lock, other instance running.") return # Lock the current state. We want to have a consistent active state. # This does communicate with the flask backend; it prevents writing and # reading to the same files at the same time. with SQLiteLock( lock_file, blocking=True, lock_name="active", project_id=project_id) as lock: # Get the all labels since last run. If no new labels, quit. new_label_history = read_label_history(project_id) data_fp = str(get_data_file_path(project_id)) as_data = read_data(project_id) state_file = get_state_path(project_id) # collect command line arguments and pass them to the reviewer with open(asr_kwargs_file, "r") as fp: asr_kwargs = json.load(fp) try: del asr_kwargs["abstract_only"] except KeyError: pass asr_kwargs['state_file'] = str(state_file) reviewer = get_reviewer(dataset=data_fp, mode="minimal", **asr_kwargs) with open_state(state_file) as state: old_label_history = _get_label_train_history(state) diff_history = _get_diff_history(new_label_history, old_label_history) if len(diff_history) == 0: logging.info( "Project {project_id} - No new labels since last run.") return query_record_ids = np.array([x[0] for x in diff_history], dtype=int) inclusions = np.array([x[1] for x in diff_history], dtype=int) query_idx = convert_id_to_idx(as_data, query_record_ids) # Classify the new labels, train and store the results. with open_state(state_file) as state: reviewer.classify( query_idx, inclusions, state, method=label_method) reviewer.train() reviewer.log_probabilities(state) new_query_idx = reviewer.query(reviewer.n_pool()).tolist() reviewer.log_current_query(state) # write the proba to a pandas dataframe with record_ids as index proba = pd.DataFrame( {"proba": state.pred_proba.tolist()}, index=pd.Index(as_data.record_ids, name="record_id") ) # update the pool and output the proba's # important: pool is sorted on query with SQLiteLock( lock_file, blocking=True, lock_name="active", project_id=project_id) as lock: # read the pool current_pool = read_pool(project_id) # diff pool and new_query_ind current_pool_idx = convert_id_to_idx(as_data, current_pool) current_pool_idx = frozenset(current_pool_idx) new_pool_idx = [x for x in new_query_idx if x in current_pool_idx] # convert new_pool_idx back to record_ids new_pool = convert_idx_to_id(as_data, new_pool_idx) # write the pool and proba write_pool(project_id, new_pool) write_proba(project_id, proba)