def api_get_prior(project_id): # noqa: F401
"""Get all papers classified as prior documents
"""
subset = request.args.get('subset', default=None, type=str)
# check if the subset exists
if subset is not None and subset not in ["included", "excluded"]:
message = "Unkown subset parameter"
return jsonify(message=message), 400
lock_fp = get_lock_path(project_id)
with SQLiteLock(lock_fp, blocking=True, lock_name="active"):
label_history = read_label_history(project_id, subset=subset)
indices = [x[0] for x in label_history]
records = read_data(project_id).record(indices)
payload = {"result": []}
for i, record in enumerate(records):
payload["result"].append({
"id": int(record.record_id),
"title": record.title,
"abstract": record.abstract,
"authors": record.authors,
"keywords": record.keywords,
"included": int(label_history[i][1])
})
response = jsonify(payload)
response.headers.add('Access-Control-Allow-Origin', '*')
return response
def label_instance(project_id, paper_i, label, retrain_model=True):
"""Label a paper after reviewing the abstract.
"""
paper_i = int(paper_i)
label = int(label)
fp_lock = get_lock_path(project_id)
with SQLiteLock(fp_lock,
blocking=True,
lock_name="active",
project_id=project_id):
# get the index of the active iteration
if int(label) in [0, 1]:
move_label_from_pool_to_labeled(project_id, paper_i, label)
else:
move_label_from_labeled_to_pool(project_id, paper_i)
if retrain_model:
# Update the model (if it isn't busy).
py_exe = _get_executable()
run_command = [py_exe, "-m", "asreview", "web_run_model", project_id]
subprocess.Popen(run_command)
def get_statistics(project_id):
fp_lock = get_lock_path(project_id)
with SQLiteLock(fp_lock, blocking=True, lock_name="active"):
# get the index of the active iteration
label_history = read_label_history(project_id)
current_labels = read_current_labels(project_id,
label_history=label_history)
n_since_last_inclusion = 0
for _, inclusion in reversed(label_history):
if inclusion == 1:
break
n_since_last_inclusion += 1
n_included = len(np.where(current_labels == 1)[0])
n_excluded = len(np.where(current_labels == 0)[0])
n_papers = len(current_labels)
stats = {
"n_included": n_included,
"n_excluded": n_excluded,
"n_since_last_inclusion": n_since_last_inclusion,
"n_papers": n_papers,
"n_pool": n_papers - n_included - n_excluded
}
return stats
def api_get_prior(project_id): # noqa: F401
"""Get all papers classified as prior documents
"""
lock_fp = get_lock_path(project_id)
with SQLiteLock(lock_fp, blocking=True, lock_name="active"):
label_history = read_label_history(project_id)
indices = [x[0] for x in label_history]
records = read_data(project_id).record(indices)
payload = {"result": []}
for i, record in enumerate(records):
payload["result"].append({
"id": int(record.record_id),
"title": record.title,
"abstract": record.abstract,
"authors": record.authors,
"keywords": record.keywords,
"included": int(label_history[i][1])
})
response = jsonify(payload)
response.headers.add('Access-Control-Allow-Origin', '*')
return response
def export_to_string(project_id, export_type="csv"):
fp_lock = get_lock_path(project_id)
as_data = read_data(project_id)
with SQLiteLock(fp_lock, blocking=True, lock_name="active"):
proba = read_proba(project_id)
if proba is None:
proba = np.flip(np.arange(len(as_data)))
else:
proba = np.array(proba)
labels = read_current_labels(project_id, as_data=as_data)
pool_idx = np.where(labels == LABEL_NA)[0]
one_idx = np.where(labels == 1)[0]
zero_idx = np.where(labels == 0)[0]
proba_order = np.argsort(-proba[pool_idx])
ranking = np.concatenate((one_idx, pool_idx[proba_order], zero_idx),
axis=None)
if export_type == "csv":
return as_data.to_csv(fp=None, labels=labels, ranking=ranking)
if export_type == "excel":
get_tmp_path(project_id).mkdir(exist_ok=True)
fp_tmp_export = Path(get_tmp_path(project_id), "export_result.xlsx")
return as_data.to_excel(fp=fp_tmp_export,
labels=labels,
ranking=ranking)
else:
raise ValueError("This export type isn't implemented.")
def add_dataset_to_project(project_id, file_name):
"""Add file path to the project file.
Add file to data subfolder and fill the pool of iteration 0.
"""
project_file_path = get_project_file_path(project_id)
fp_lock = get_lock_path(project_id)
with SQLiteLock(fp_lock, blocking=True, lock_name="active"):
# open the projects file
with open(project_file_path, "r") as f_read:
project_dict = json.load(f_read)
# add path to dict (overwrite if already exists)
project_dict["dataset_path"] = file_name
with open(project_file_path, "w") as f_write:
json.dump(project_dict, f_write)
# fill the pool of the first iteration
pool_indices = read_data(project_id).record_ids
np.random.shuffle(pool_indices)
write_pool(project_id, pool_indices.tolist())
# make a empty qeue for the items to label
write_label_history(project_id, [])
def remove_dataset_to_project(project_id, file_name):
"""Remove dataset from project
"""
project_file_path = get_project_file_path(project_id)
fp_lock = get_lock_path(project_id)
with SQLiteLock(fp_lock,
blocking=True,
lock_name="active",
project_id=project_id):
# open the projects file
with open(project_file_path, "r") as f_read:
project_dict = json.load(f_read)
# remove the path from the project file
data_fn = project_dict["dataset_path"]
del project_dict["dataset_path"]
with open(project_file_path, "w") as f_write:
json.dump(project_dict, f_write)
# files to remove
data_path = get_data_file_path(project_id, data_fn)
pool_path = get_pool_path(project_id)
labeled_path = get_labeled_path(project_id)
os.remove(str(data_path))
os.remove(str(pool_path))
os.remove(str(labeled_path))
def get_statistics(project_id):
"""Get statistics from project files.
Arguments
---------
project_id: str
The id of the current project.
Returns
-------
dict:
Dictonary with statistics.
"""
fp_lock = get_lock_path(project_id)
with SQLiteLock(fp_lock,
blocking=True,
lock_name="active",
project_id=project_id):
# get the index of the active iteration
labeled = read_label_history(project_id)
pool = read_pool(project_id)
# compute metrics
n_included = n_relevant(labeled)
n_excluded = n_irrelevant(labeled)
n_pool = len(pool)
return {
"n_included": n_included,
"n_excluded": n_excluded,
"n_since_last_inclusion": stop_n_since_last_relevant(labeled),
"n_papers": n_pool + n_included + n_excluded,
"n_pool": n_pool
}
def export_to_string(project_id, export_type="csv"):
# read the dataset into a ASReview data object
as_data = read_data(project_id)
# set the lock to safely read labeled, pool, and proba
fp_lock = get_lock_path(project_id)
with SQLiteLock(
fp_lock,
blocking=True,
lock_name="active",
project_id=project_id
):
proba = read_proba(project_id)
pool = read_pool(project_id)
labeled = read_label_history(project_id)
# get the record_id of the inclusions and exclusions
inclusion_record_id = [int(x[0]) for x in labeled if x[1] == 1]
exclusion_record_id = [int(x[0]) for x in labeled if x[1] == 0]
# order the pool from high to low proba
if proba is not None:
pool_ordered = proba.loc[pool, :] \
.sort_values("proba", ascending=False).index.values
else:
pool_ordered = pool_ordered
# get the ranking of the 3 subcategories
ranking = np.concatenate(
(
# add the inclusions first
inclusion_record_id,
# add the ordered pool second
pool_ordered,
# add the exclusions last
exclusion_record_id
),
axis=None
)
# export the data to file
if export_type == "csv":
return as_data.to_csv(fp=None, labels=labeled, ranking=ranking)
if export_type == "tsv":
return as_data.to_csv(
fp=None, sep="\t", labels=labeled, ranking=ranking)
if export_type == "excel":
get_tmp_path(project_id).mkdir(exist_ok=True)
fp_tmp_export = Path(get_tmp_path(project_id), "export_result.xlsx")
return as_data.to_excel(
fp=fp_tmp_export, labels=labeled, ranking=ranking)
else:
raise ValueError("This export type isn't implemented.")
def api_random_prior_papers(project_id): # noqa: F401
"""Get a selection of random papers to find exclusions.
This set of papers is extracted from the pool, but without
the already labeled items.
"""
lock_fp = get_lock_path(project_id)
with SQLiteLock(lock_fp,
blocking=True,
lock_name="active",
project_id=project_id):
pool = read_pool(project_id)
# with open(get_labeled_path(project_id, 0), "r") as f_label:
# prior_labeled = json.load(f_label)
# excluded the already labeled items from our random selection.
# prior_labeled_index = [int(label) for label in prior_labeled.keys()]
# pool = [i for i in pool if i not in prior_labeled_index]
# sample from the pool (this is already done atm of initializing
# the pool. But doing it again because a double shuffle is always
# best)
try:
pool_random = np.random.choice(pool, 1, replace=False)[0]
except Exception:
raise ValueError("Not enough random indices to sample from.")
try:
record = read_data(project_id).record(pool_random, by_index=False)
payload = {"result": []}
payload["result"].append({
"id": int(record.record_id),
"title": record.title,
"abstract": record.abstract,
"authors": record.authors,
"keywords": record.keywords,
"included": None
})
except Exception as err:
logging.error(err)
return jsonify(message="Failed to load random documents."), 500
response = jsonify(payload)
response.headers.add('Access-Control-Allow-Origin', '*')
return response
def add_dataset_to_project(project_id, file_name):
"""Add file path to the project file.
Add file to data subfolder and fill the pool of iteration 0.
"""
project_file_path = get_project_file_path(project_id)
# clean temp project files
clean_project_tmp_files(project_id)
with SQLiteLock(
get_lock_path(project_id),
blocking=True,
lock_name="active",
project_id=project_id
):
# open the projects file
with open(project_file_path, "r") as f_read:
project_dict = json.load(f_read)
# add path to dict (overwrite if already exists)
project_dict["dataset_path"] = file_name
with open(project_file_path, "w") as f_write:
json.dump(project_dict, f_write)
# fill the pool of the first iteration
as_data = read_data(project_id)
if as_data.labels is not None:
unlabeled = np.where(as_data.labels == LABEL_NA)[0]
pool_indices = as_data.record_ids[unlabeled]
labeled_indices = np.where(as_data.labels != LABEL_NA)[0]
label_indices = list(zip(
as_data.record_ids[labeled_indices].tolist(),
as_data.labels[labeled_indices].tolist()
))
else:
pool_indices = as_data.record_ids
label_indices = []
np.random.shuffle(pool_indices)
write_pool(project_id, pool_indices.tolist())
# make a empty qeue for the items to label
write_label_history(project_id, label_indices)
def get_instance(project_id):
"""Get a new instance to review.
Arguments
---------
project_id: str
The id of the current project.
"""
fp_lock = get_lock_path(project_id)
with SQLiteLock(fp_lock, blocking=True, lock_name="active"):
pool_idx = read_pool(project_id)
logging.info(f"Requesting {pool_idx[0]} from project {project_id}")
return pool_idx[0]
def api_get_prior_stats(project_id): # noqa: F401
"""Get all papers classified as prior documents
"""
lock_fp = get_lock_path(project_id)
with SQLiteLock(lock_fp, blocking=True, lock_name="active"):
label_history = read_label_history(project_id)
counter_prior = Counter([x[1] for x in label_history])
response = jsonify({
"n_prior": len(label_history),
"n_inclusions": counter_prior[1],
"n_exclusions": counter_prior[0]
})
response.headers.add('Access-Control-Allow-Origin', '*')
return response
def export_to_string(project_id):
fp_lock = get_lock_path(project_id)
as_data = read_data(project_id)
with SQLiteLock(fp_lock, blocking=True, lock_name="active"):
proba = read_proba(project_id)
if proba is None:
proba = np.flip(np.arange(len(as_data)))
else:
proba = np.array(proba)
labels = read_current_labels(project_id, as_data=as_data)
pool_idx = np.where(labels == LABEL_NA)[0]
one_idx = np.where(labels == 1)[0]
zero_idx = np.where(labels == 0)[0]
proba_order = np.argsort(-proba[pool_idx])
ranking = np.concatenate((one_idx, pool_idx[proba_order], zero_idx),
axis=None)
return as_data.to_csv(fp=None, labels=labels, ranking=ranking)
def get_instance(project_id):
"""Get a new instance to review.
Arguments
---------
project_id: str
The id of the current project.
"""
fp_lock = get_lock_path(project_id)
with SQLiteLock(
fp_lock, blocking=True, lock_name="active", project_id=project_id):
pool_idx = read_pool(project_id)
if len(pool_idx) > 0:
return pool_idx[0]
else:
# end of pool
return None
def add_dataset_to_project(project_id, file_name):
"""Add file path to the project file.
Add file to data subfolder and fill the pool of iteration 0.
"""
project_file_path = get_project_file_path(project_id)
fp_lock = get_lock_path(project_id)
with SQLiteLock(fp_lock, blocking=True, lock_name="active"):
# open the projects file
with open(project_file_path, "r") as f_read:
project_dict = json.load(f_read)
# add path to dict (overwrite if already exists)
project_dict["dataset_path"] = file_name
with open(project_file_path, "w") as f_write:
json.dump(project_dict, f_write)
# fill the pool of the first iteration
as_data = read_data(project_id)
if as_data.labels is not None:
unlabeled = np.where(as_data.labels == LABEL_NA)[0]
pool_indices = as_data.record_ids[unlabeled]
label_indices_included = \
[[int(x), 1] for x in np.where(as_data.labels == 1)[0]]
label_indices_excluded = \
[[int(x), 0] for x in np.where(as_data.labels == 0)[0]]
label_indices = label_indices_included + label_indices_excluded
else:
pool_indices = as_data.record_ids
label_indices = []
np.random.shuffle(pool_indices)
write_pool(project_id, pool_indices.tolist())
# make a empty qeue for the items to label
write_label_history(project_id, label_indices)
def label_instance(project_id, paper_i, label, retrain_model=True):
"""Label a paper after reviewing the abstract.
"""
paper_i = int(paper_i)
label = int(label)
fp_lock = get_lock_path(project_id)
with SQLiteLock(fp_lock, blocking=True, lock_name="active"):
# get the index of the active iteration
if int(label) in [0, 1]:
move_label_from_pool_to_labeled(project_id, paper_i, label)
else:
move_label_from_labeled_to_pool(project_id, paper_i, label)
if retrain_model:
# Update the model (if it isn't busy).
run_command = f"python -m asreview web_run_model '{project_id}'"
subprocess.Popen(shlex.split(run_command))
def api_get_prior_stats(project_id): # noqa: F401
"""Get all papers classified as prior documents
"""
try:
lock_fp = get_lock_path(project_id)
with SQLiteLock(lock_fp,
blocking=True,
lock_name="active",
project_id=project_id):
label_history = read_label_history(project_id)
counter_prior = Counter([x[1] for x in label_history])
except Exception as err:
logging.error(err)
return jsonify(message="Failed to load prior information."), 500
response = jsonify({
"n_prior": len(label_history),
"n_inclusions": counter_prior[1],
"n_exclusions": counter_prior[0]
})
response.headers.add('Access-Control-Allow-Origin', '*')
return response
def train_model(project_id, label_method=None):
"""Add the new labels to the review and do the modeling.
It uses a lock to ensure only one model is running at the same time.
Old results directories are deleted after 4 iterations.
It has one argument on the CLI, which is the base project directory.
"""
logging.info(f"Project {project_id} - Train a new model for project")
# get file locations
asr_kwargs_file = get_kwargs_path(project_id)
lock_file = get_lock_path(project_id)
# Lock so that only one training run is running at the same time.
# It doesn't lock the flask server/client.
with SQLiteLock(
lock_file, blocking=False, lock_name="training",
project_id=project_id) as lock:
# If the lock is not acquired, another training instance is running.
if not lock.locked():
logging.info("Project {project_id} - "
"Cannot acquire lock, other instance running.")
return
# Lock the current state. We want to have a consistent active state.
# This does communicate with the flask backend; it prevents writing and
# reading to the same files at the same time.
with SQLiteLock(
lock_file,
blocking=True,
lock_name="active",
project_id=project_id) as lock:
# Get the all labels since last run. If no new labels, quit.
new_label_history = read_label_history(project_id)
data_fp = str(get_data_file_path(project_id))
as_data = read_data(project_id)
state_file = get_state_path(project_id)
# collect command line arguments and pass them to the reviewer
with open(asr_kwargs_file, "r") as fp:
asr_kwargs = json.load(fp)
try:
del asr_kwargs["abstract_only"]
except KeyError:
pass
asr_kwargs['state_file'] = str(state_file)
reviewer = get_reviewer(dataset=data_fp, mode="minimal", **asr_kwargs)
with open_state(state_file) as state:
old_label_history = _get_label_train_history(state)
diff_history = _get_diff_history(new_label_history, old_label_history)
if len(diff_history) == 0:
logging.info(
"Project {project_id} - No new labels since last run.")
return
query_record_ids = np.array([x[0] for x in diff_history], dtype=int)
inclusions = np.array([x[1] for x in diff_history], dtype=int)
query_idx = convert_id_to_idx(as_data, query_record_ids)
# Classify the new labels, train and store the results.
with open_state(state_file) as state:
reviewer.classify(
query_idx, inclusions, state, method=label_method)
reviewer.train()
reviewer.log_probabilities(state)
new_query_idx = reviewer.query(reviewer.n_pool()).tolist()
reviewer.log_current_query(state)
# write the proba to a pandas dataframe with record_ids as index
proba = pd.DataFrame(
{"proba": state.pred_proba.tolist()},
index=pd.Index(as_data.record_ids, name="record_id")
)
# update the pool and output the proba's
# important: pool is sorted on query
with SQLiteLock(
lock_file,
blocking=True,
lock_name="active",
project_id=project_id) as lock:
# read the pool
current_pool = read_pool(project_id)
# diff pool and new_query_ind
current_pool_idx = convert_id_to_idx(as_data, current_pool)
current_pool_idx = frozenset(current_pool_idx)
new_pool_idx = [x for x in new_query_idx if x in current_pool_idx]
# convert new_pool_idx back to record_ids
new_pool = convert_idx_to_id(as_data, new_pool_idx)
# write the pool and proba
write_pool(project_id, new_pool)
write_proba(project_id, proba)
def train_model(project_id, label_method=None):
"""Add the new labels to the review and do the modeling.
It uses a lock to ensure only one model is running at the same time.
Old results directories are deleted after 4 iterations.
It has one argument on the CLI, which is the base project directory.
"""
logging.info(f"Project {project_id} - Train a new model for project")
# get file locations
asr_kwargs_file = get_kwargs_path(project_id)
lock_file = get_lock_path(project_id)
# Lock so that only one training run is running at the same time.
# It doesn't lock the flask server/client.
with SQLiteLock(lock_file,
blocking=False,
lock_name="training",
project_id=project_id) as lock:
# If the lock is not acquired, another training instance is running.
if not lock.locked():
logging.info("Project {project_id} - "
"Cannot acquire lock, other instance running.")
return
# Lock the current state. We want to have a consistent active state.
# This does communicate with the flask backend; it prevents writing and
# reading to the same files at the same time.
with SQLiteLock(lock_file,
blocking=True,
lock_name="active",
project_id=project_id) as lock:
# Get the all labels since last run. If no new labels, quit.
new_label_history = read_label_history(project_id)
data_fp = str(get_data_file_path(project_id))
as_data = read_data(project_id)
state_file = get_state_path(project_id)
# collect command line arguments and pass them to the reviewer
with open(asr_kwargs_file, "r") as fp:
asr_kwargs = json.load(fp)
asr_kwargs['state_file'] = str(state_file)
reviewer = get_reviewer(dataset=data_fp, mode="minimal", **asr_kwargs)
with open_state(state_file) as state:
old_label_history = get_label_train_history(state)
diff_history = get_diff_history(new_label_history, old_label_history)
if len(diff_history) == 0:
logging.info(
"Project {project_id} - No new labels since last run.")
return
query_idx = np.array([x[0] for x in diff_history], dtype=int)
inclusions = np.array([x[1] for x in diff_history], dtype=int)
# Classify the new labels, train and store the results.
with open_state(state_file) as state:
reviewer.classify(query_idx,
inclusions,
state,
method=label_method)
reviewer.train()
reviewer.log_probabilities(state)
new_query_idx = reviewer.query(reviewer.n_pool()).tolist()
reviewer.log_current_query(state)
proba = state.pred_proba.tolist()
with SQLiteLock(lock_file,
blocking=True,
lock_name="active",
project_id=project_id) as lock:
current_pool = read_pool(project_id)
in_current_pool = np.zeros(len(as_data))
in_current_pool[current_pool] = 1
new_pool = [x for x in new_query_idx if in_current_pool[x]]
write_pool(project_id, new_pool)
write_proba(project_id, proba)
