def login():
"""
Logs into LORIS using the stored credential. Must use PyCurl as Requests is not working.
:return: BOOL if or not it is successful. also, the JSON token that is necessary to conduct further transactions.
"""
from DICOMTransit.settings import config_get
username = config_get("LORISusername")
password = config_get("LORISpassword")
data = json.dumps({"username": username, "password": password})
# Login URL
url = config_get("LORISurl")
if type(url) is furl:
updated_url = url.url + "login"
else:
updated_url = url + "login"
# requests style login # NOT WORKING!
r = requests.post(updated_url, data=data)
logger.debug(str(r.status_code) + r.reason)
response_json = r.json()
return LORIS_helper.is_response(r.status_code, 200), response_json.get("token")
def get_dev_orthanc_credentials() -> orthanc_credential:
"""
Obtain the Development Orthanc instance credential
:return:
"""
from DICOMTransit.settings import config_get
url = config_get("DevOrthancIP")
user = config_get("DevOrthancUser")
password = config_get("DevOrthancPassword")
return orthanc_credential(url, user, password)
def get_prod_orthanc_credentials() -> orthanc_credential:
"""
Obtain the Production Orthanc instance credential
:return:
"""
from DICOMTransit.settings import config_get
url = config_get("ProdOrthancIP")
user = config_get("ProdOrthancUser")
password = config_get("ProdOrthancPassword")
return orthanc_credential(url, user, password)
def append_StudyUID(MRN: int, StudyUID: str):
"""
Update record with proper completion status which STUDY has particular MRN
:param MRN:
:return:
"""
# convert new study UID to list.
list_StudyUID_single_new = [StudyUID]
database_path = config_get("LocalDatabasePath")
# obtain
list_StudyUID_existing_ones = get_StudyUIDs(MRN)
if list_StudyUID_existing_ones is None:
list_StudyUID_total = list_StudyUID_single_new
else:
list_StudyUID_total = list_StudyUID_existing_ones + list_StudyUID_single_new
# JSON dumps.
json_seriesUID = json.dumps(list_StudyUID_total)
# Update the MRN record with StudyUID
LocalDB_query.update_entry(
database_path,
CNBP_blueprint.table_name,
CNBP_blueprint.keyfield,
MRN,
"StudyUID",
json_seriesUID,
)
def putCNBPDICOM(token, endpoint, imaging_data, isPhantom: bool = False):
"""
Put some data to a LORIS end point.
:param token:
:param endpoint:
:param imaging_data:
:param isPhantom: whether the upload is a phantom data or not.
:return: bool on if request is successful, r for the request (CAN BE NULL for 201 based requests)
"""
logger.debug(f"Uploading Imaging data to: {endpoint}")
url = config_get("LORISurl")
updatedurl = url + endpoint
if isPhantom:
HEADERS = {"Authorization": f"token {token}", "X-Is-Phantom": "1"}
else:
HEADERS = {"Authorization": f"token {token}", "X-Is-Phantom": "0"}
with requests.Session() as s:
s.headers.update(HEADERS)
r = s.put(updatedurl, data=imaging_data)
logger.debug(f"Put Result: {str(r.status_code)} {r.reason}")
return r.status_code, r
def get_CNNID(MRN: int) -> Optional[List[str]]:
"""
Assuming the MRN exist, get the CNNIDs of all scans that ever past through here.
:param MRN: the MRN to look for
:return: the CNBPID associated with that particular MRN number.
"""
# Load local database from .env file
database_path = config_get("LocalDatabasePath")
MRN_exist_in_database, KeyRecords = LocalDB_query.check_value(
database_path, CNBP_blueprint.table_name, CNBP_blueprint.keyfield, MRN)
if MRN_exist_in_database is False:
return None
# Only ONE record per MRN.
assert len(KeyRecords) == 1
CNNID_index = LocalDB_query.check_header_index(database_path,
CNBP_blueprint.table_name,
"CNNID")
# Load the json and return the variable structure
json_CNNIDs = KeyRecords[0][CNNID_index]
if json_CNNIDs is None:
logger.warning("No existing CNNID Data information found.")
return None
list_CNNIDs = json.loads(json_CNNIDs)
return list_CNNIDs
def trigger_dicom_insert(scans):
"""
Triggers insertion of uploaded dicom files that are not yet inserted into
Loris.
:param scans: An array of dictionaries that represent the params for each
dicom file to insert
:returns:
"""
# @todo: 2018-11-30T133155EST should clean up here and ensure loading the URL using .env.
# Create a dictionary with the required key 'dicoms'. Key value is scans
p = {"dicoms": scans}
# Convert to json.
# See https://pythonspot.com/json-encoding-and-decoding-with-python/
tmp = json.dumps(p, ensure_ascii=False)
# Put json into dictionary as a value of the required key 'file_data'
payload = {"file_data": tmp}
print(payload)
# Need to fix the load_dotenv call. Currently not getting trigger url
trigger_dicom_insertion_url = config_get("InsertionAPI")
# Trigger insertion by doing HTTP POST of payload to endpoint
s = requests.post(trigger_dicom_insertion_url, data=payload)
# print(trigger_dicom_insertion_url)
print(s.text)
def append_SeriesUID(MRN: int, SeriesUID: List[str]):
"""
Append record with SeriesUID list. which has particular MRN
:param MRN:
:return:
"""
database_path = config_get("LocalDatabasePath")
existing_series_UID = get_SeriesUIDs(MRN)
if existing_series_UID is None:
total_series_UID = SeriesUID
else:
total_series_UID = existing_series_UID + SeriesUID
# JSON dumps.
json_seriesUID = json.dumps(total_series_UID)
# Update the MRN record with SeriesUID
LocalDB_query.update_entry(
database_path,
CNBP_blueprint.table_name,
CNBP_blueprint.keyfield,
MRN,
"SeriesUID",
json_seriesUID,
)
def increaseTimepoint(token: str, DCCID: int) -> (bool, str):
"""
Increment the existing timepoint by check if DCCID existed, then get the latest one, then increment its number by creating a new timepoint.
:param token: auth token
:param DCCID: the DCCID of the existing subject.
:return: if creation request is successful, and what label is actually created.
"""
# @todo: must handle the special edge case where timepoint reach V10 etc where two or three more digits are there!
timepoint_label = ""
# ensure valid input and subject actually exist.
assert LORIS_validation.validate_DCCID(DCCID)
subject_exist, _ = LORIS_candidates.checkDCCIDExist(token, DCCID)
if not subject_exist:
return False, None
latest_timepoint = LORIS_timepoint.findLatestTimePoint(token, DCCID)
if latest_timepoint is None:
success = LORIS_timepoint.createTimepoint(token, DCCID, "V1")
timepoint_label = "V1"
else:
visit_number = LORIS_timepoint.visit_number_extraction(
latest_timepoint)
new_visit_number = int_incrementor(visit_number)
prefix = config_get("timepoint_prefix")
timepoint_label = prefix + str(new_visit_number)
success = LORIS_timepoint.createTimepoint(token, DCCID,
timepoint_label)
return success, timepoint_label
def UnpackNewData(self):
"""
Properly create the DICOM package and update its relevant basic informations like files reference and UIDs
:return:
"""
path_temp_zip = config_get("ZipPath")
# Properly set the DICOM_package.
temporary_folder = DICOMTransit.orthanc.API.unpack_subject_zip(
self.DICOM_zip, path_temp_zip)
# Orthanc files are GUARNTEED to have consistency name and ID so no need to check that. A cursory DICOM check is good enough for performance reasons.
self.DICOM_package = DICOMPackage(
temporary_folder,
consistency_check=False) # Series UID is extracted at this time.
# Update unique UID information to help discriminate existing scans.
# self.DICOM_package.update_sUID()
self.DICOM_package.project = (
"loris"
) # fixme: this is a place holder. This neeeds to be dyanmiclly updated.
# Update the self.files to be scrutinized
self.files.clear()
self.files = self.DICOM_package.get_dicom_files(
consistency_check=False) # consistency do not need to be checked.
logger.info("Successfully unpacked the data downloaded.")
def test_env(self):
"""
This unit test ensures that the variables in .env is the same as the ones specified in the blueprint of the schema
:return:
"""
# Load the list of predefined variables as defined in the schema.
list_variables = CNBP_blueprint.dotenv_variables
# loop through each one and then attempt to load and validate them.
for variable in list_variables:
assert config_get(variable) is not None
def change_to_zip_dir():
# Load the name of the storage folder from the configuration file.
folder_to_zip = config_get("ZipPath")
# Find the root fo the project where the zip storage is related to the location of the current DICOM directory.
project_root = get_abspath(__file__, 2)
# Create absolute path to the folder to be zipped
zip_path = os.path.join(project_root, folder_to_zip)
#
os.chdir(zip_path)
def old_upload(local_path):
"""
OBSOLETE Upload file to incoming folder.
:param local_path:
:return:
"""
ProxyIP = config_get("ProxyIP")
ProxyUsername = config_get("ProxyUsername")
ProxyPassword = config_get("ProxyPassword")
LORISHostIP = config_get("LORISHostIP")
LORISHostUsername = config_get("LORISHostUsername")
LORISHostPassword = config_get("LORISHostPassword")
Client = LORIS_helper.getProxySSHClient(
ProxyIP,
ProxyUsername,
ProxyPassword,
LORISHostIP,
LORISHostUsername,
LORISHostPassword,
)
file_name = os.path.basename(local_path)
LORIS_helper.uploadThroughClient(Client, "//data/incoming/" + file_name,
local_path)
def propose_CNBPID(DICOM_protocol: str):
"""
This function takes in a string that is representative of the DICOM acquisition study protocol, and propose a CNBPID composed of two parts:
Institution_ID (from the .env configuration file)
Project_ID (inferred from the protocol and incremented
SubjectCount (kept track by localDB.
:param DICOM_protocol:
:return:
"""
# Get and retrieve institution_ID
InstitionID = config_get("institutionID")
import DICOMTransit.DICOM.API
# Check ProjectID string and then return the ProjectID;
ProjectID = DICOMTransit.DICOM.API.study_validation(DICOM_protocol)
# Use those two pieces of information to form a partial query pattern that can run on the SQLite
partial_search_input = InstitionID + ProjectID
DB_path = config_get("LocalDatabasePath")
# Partial match search records: (currently has SQL error).
success, matched_records = LocalDB_query.check_partial_value(
DB_path, CNBP_blueprint.table_name, "CNBPID", partial_search_input)
# Default subject ID when no match is found.
latest_subject_ID = "0000001"
if matched_records is None or len(matched_records) == 0:
# no previous subjects found. Use default value.
pass
else:
latest_subject_ID = check_all_existing_records(matched_records)
# Combined all the parts to return the proposed CNBPID
proposed_CNBPID = InstitionID + ProjectID + latest_subject_ID
return proposed_CNBPID
def validate_instutitionID(input_institutionID: str) -> bool:
"""
Check if the institution ID is compliant per the .env specification. String must STRICTLY match.
:param input_institutionID:
:return:
"""
# Parse from the .env standardization
InsitituionID = config_get("institutionID")
# Check if institution ID matches
if not (input_institutionID == InsitituionID):
return False
else:
return True
def deleteCandidateCNBP(DCCID, PSCID):
# @todo: this should really be done through API. But Currently LORIS does not offer such API.
# NOTE! If you EVER get NULL coalesce not recognized error, make sure that the PHP version being called from
# the SSH session is 7+ or else. We had a major issue where the PHP version from SSH session being LOWER
# than the .bashrc profile imported edition. Also keep in mind that EVEN if .bashrc import this, it MOST LIKELY
# will not apply to the SSH session!
LORISHostPassword = config_get("LORISHostPassword")
LORISHostUsername = config_get("LORISHostUsername")
LORISHostIP = config_get("LORISHostIP")
DeletionScript = config_get("DeletionScript")
# NOTE! If you EVER get NULL coalesce not recognized error, make sure that the PHP version being called from
# the SSH session is 7+ or else. We had a major issue where the PHP version from SSH session being LOWER
# than the bashrc profile imported edition. Also keep in mind that EVEN if .bashrc import this, it MOST LIKELY
# will not apply to the SSH session!
command_string = f"/opt/rh//rh-php70/root/usr/bin/php {DeletionScript} delete_candidate {str(DCCID)} {PSCID} confirm"
logger.debug(command_string)
# Establish connection to client.
Client = LORIS_helper.getProxySSHClient(
ProxyIP,
ProxyUsername,
ProxyPassword,
LORISHostIP,
LORISHostUsername,
LORISHostPassword,
)
# Execute the command
LORIS_helper.triggerCommand(Client, command_string)
# Close the client.
Client.close()
def check_MRN(MRN: int) -> bool:
"""
Return true if the MRN exist within the current database
:return:
"""
# Load local database from .env file
database_path = config_get("LocalDatabasePath")
# Store MRN in database.
MRN_exist_in_database, _ = LocalDB_query.check_value(
database_path, CNBP_blueprint.table_name, CNBP_blueprint.keyfield, MRN)
if MRN_exist_in_database:
logger.info("MRN found to exist at local database")
return MRN_exist_in_database
def checkPSCIDExist(token, proposed_PSCID):
"""
Check if Site/Study already contain the PSCID
@todo: mostly obsolete as now PSCID is completely generated server side.
HOWEVER! THere are /candidate_list that can once for all get
:param token:
:param proposed_PSCID:
:return: bool for connection, bool on if such PSCID (INSTITUTIONID + PROJECTID + SUBJECTID) exist already.
"""
logger.debug("Checking if PSCID exist: " + proposed_PSCID)
institution_check = config_get("institutionID")
# Get list of projects
response_success, loris_project = LORIS_query.getCNBP(
token, r"projects/loris")
if not response_success:
return response_success, None
# Get list of candidates (Candidates in v0.0.1)
candidates = loris_project.get("Candidates")
logger.debug(candidates)
for (DCCID) in (
candidates
): # these candidates should really be only from the same ID regions.
response_success, candidate_json = LORIS_query.getCNBP(
token, r"candidates/" + DCCID)
if not response_success:
return response_success, False
# Each site following the naming convention should have SITE prefix and PI prefix and PROJECT prefix to avoid collision
# A site check here.
candidate_meta = candidate_json.get("Meta")
candidate_pscID = candidate_meta.get("PSCID")
# Not gonna check is institution ID doesn't even match.
if candidate_pscID[0:3] != institution_check:
continue
elif candidate_pscID == proposed_PSCID:
return response_success, True
# latest_timepoint = findLatestTimePoint(DCCID)
return True, False
def createCandidate(token, project, birth_date, gender):
"""
Create a candidate with a provided project information. This assumes that the PSCID and DCCIDs are automaticly assigned!
:param token
:param birth_date: Birth date MUST Be in YYYY-MM-DD format!
:param gender: Gender must be Male or Female!
:param project:
:return: DCCID
"""
logger.debug(f"Creating CNBP Candidates belong to project: {project}")
Candidate = {}
from DICOMTransit.LORIS.validate import LORIS_validation
if not LORIS_validation.validate_birth_date_loris(
birth_date
) or not LORIS_validation.validate_gender(
gender
): # not LORIS_validation.validate_project(project) or #@todo fix this project validation part during creation.
logger.error(
"Non-compliant PSCID component detected. Aborting PSCID creation "
)
return False, None
Candidate["Project"] = project
Candidate["EDC"] = birth_date
# Candidate['PSCID'] = proposed_PSCID Now auto sequence.
Candidate["DoB"] = birth_date
Candidate["Gender"] = gender
Candidate["Site"] = config_get("institutionName")
data = {"Candidate": Candidate}
data_json = json.dumps(data)
response_code, response = LORIS_query.postCNBP(token, "candidates/",
data_json)
if not LORIS_helper.is_response(response_code, 201):
return False, None, None
elif response is not None: # only try to decode if response is not empty!
response_json = response.json()
meta = response_json.get("Meta")
CandID = meta.get("CandID")
return True, CandID
else:
return False, None
def set_DCCID(MRN: int, DCCID: int):
"""
Update record with proper DCCID which has particular MRN
:param MRN:
:return:
"""
database_path = config_get("LocalDatabasePath")
# Update the MRN record with DCCID
LocalDB_query.update_entry(
database_path,
CNBP_blueprint.table_name,
CNBP_blueprint.keyfield,
MRN,
"DCCID",
DCCID,
)
def set_timepoint(MRN: int, Timepoint: str):
"""
Update record with proper Timepoint which has particular MRN
:param MRN:
:return:
"""
database_path = config_get("LocalDatabasePath")
# Update the MRN record with Timepoint
LocalDB_query.update_entry(
database_path,
CNBP_blueprint.table_name,
CNBP_blueprint.keyfield,
MRN,
"Timepoint",
Timepoint,
)
def zip(self):
"""
:return:
"""
# load system default ZIP storage path.
zip_storage_path = config_get("ZipPath")
# Change to the storage folder before carrying out the zip operation.
os.chdir(zip_storage_path)
zip_with_name(self.dicom_folder, self.zipname)
# update zip location, this is the ABSOLUTE path.
self.zip_location = os.path.join(zip_storage_path,
self.zipname + ".zip")
self.is_zipped = True
def get_list_CNBPID() -> List[str]:
"""
Return a list of all CNBPID (Key) from the database.
:return:
"""
# Load local database from .env file
database_path = config_get("LocalDatabasePath")
list_CNBPID = []
success, result_rows = LocalDB_query.get_all(database_path,
CNBP_blueprint.table_name,
"CNBPID")
for row in result_rows:
list_CNBPID.append(row[0]) # MRN is the first variable requested.
return list_CNBPID
def get_scan_date(MRN: int) -> Optional[str]:
"""
Assuming the MRN exist, get the MRNID.
:param MRN: the MRN to look for
:return: the CNBPID associated with that particular MRN number.
"""
# Load local database from .env file
database_path = config_get("LocalDatabasePath")
MRN_exist_in_database, KeyRecords = LocalDB_query.check_value(
database_path, CNBP_blueprint.table_name, CNBP_blueprint.keyfield, MRN)
if MRN_exist_in_database is False:
return None
# Only ONE record per MRN, even if there are multiple timepoint. We keep the latest one.
assert len(KeyRecords) == 1
date_header_index = LocalDB_query.check_header_index(
database_path, CNBP_blueprint.table_name, "Date")
scan_date = KeyRecords[0][date_header_index]
return scan_date
def get_DCCID(MRN: int) -> Optional[str]:
"""
Assuming the MRN exist, get the MRNID.
:param MRN: the MRN to look for
:return: the CNBPID associated with that particular MRN number.
"""
# Load local database from .env file
database_path = config_get("LocalDatabasePath")
MRN_exist_in_database, KeyRecords = LocalDB_query.check_value(
database_path, CNBP_blueprint.table_name, CNBP_blueprint.keyfield, MRN)
if MRN_exist_in_database is False:
return None
# Only ONE record per MRN.
assert len(KeyRecords) == 1
dcc_header_index = LocalDB_query.check_header_index(
database_path, CNBP_blueprint.table_name, "DCCID")
return KeyRecords[0][dcc_header_index]
def validate_projectID(input_projectID: str) -> bool:
"""
Provide any string, and it checkss again he dotenv for the proper project KEY which correspond to the ID.
DICOM/API has the function to actually retrieve the relevant key, after calling this function.
:param input_projectID:
:return:
"""
# Load ProjectIDs from the environment.
projectID_dictionary_json: str = config_get("projectID_dictionary")
projectID_list = json.loads(projectID_dictionary_json)
# check if project ID is in the projectID list via a dictionary search
key = dictionary_search(projectID_list, input_projectID)
if key is not None:
return True
else:
return False
def set_CNNIDs(MRN: int, CaseIDs: List[int]):
"""
Update record with proper CNN which has particular MRN
:param MRN:
:return:
"""
database_path = config_get("LocalDatabasePath")
# JSON dumps.
json_CaseIDs = json.dumps(CaseIDs)
# Update the MRN record with DCCID
LocalDB_query.update_entry(
database_path,
CNBP_blueprint.table_name,
CNBP_blueprint.keyfield,
MRN,
"CNNID",
json_CaseIDs,
)
def get_list_MRN() -> List[int]:
"""
Return a list_return of all MRN from the database.
:return:
"""
# Load local database from .env file
database_path = config_get("LocalDatabasePath")
list_MRN = []
success, result_rows = LocalDB_query.get_all(database_path,
CNBP_blueprint.table_name,
"MRN")
for row in result_rows:
list_MRN.append(row[0]) # MRN is the first variable requested.
return (
list_MRN
) # @todo: verify this is in integer? or string as that has dire consequences.
def study_validation(study: str) -> str:
"""
Given a string read from the DICOM studies field, check it against the project ID dictionary to see if any of the project belongs.
:param study:
:return: PROJECT or NONE
"""
import json
# It checks if the key exist first.
if LORIS_validation.validate_projectID(study) is False:
return False
projectID_dictionary_json: str = config_get("projectID_dictionary")
projectID_list = json.loads(projectID_dictionary_json)
# check if project ID is in the projectID list.
key = dictionary_search(projectID_list, study)
# @todo: what if the key does not exist?
return key
def getCNBP(token, endpoint):
"""
Get from a CNBP LORIS database endpoint
:param token:
:param endpoint:
:return: bool on if such PSCID (INSTITUTIONID + PROJECTID + SUBJECTID) exist already.
"""
logger.debug(f"Getting LORIS endpoint: {endpoint} at")
url = config_get("LORISurl")
updatedurl = urljoin(url, endpoint)
logger.debug(updatedurl)
HEADERS = {"Authorization": "token {}".format(token)}
with requests.Session() as s:
s.headers.update(HEADERS)
r = s.get(updatedurl)
logger.debug(f"Get Result: {str(r.status_code)} {r.reason}")
return r.status_code, r.json()
