class NAACCR_Ontology2(_RunScriptTask):
'''NAACCR Ontology: un-published code values
i.e. code values that occur in tumor registry data but not in published ontologies
'''
# flat file attributes
dateCaseReportExported = pv.DateParam()
npiRegistryId = pv.StrParam()
source_cd = pv.StrParam(default='*****@*****.**')
z_design_id = pv.StrParam(default='length 50 (%s)' % _stable_hash(
tr_ont.NAACCR_I2B2.ont_script.code,
td.DataSummary.script.code))
script_name = 'naaccr_concepts_mix.sql'
script = res.read_text(heron_load, script_name)
@property
def classpath(self) -> str:
return self.jdbc_driver_jar
def _upload_target(self) -> 'UploadTarget':
return UploadTarget(self, self.schema, transform_name=self.task_id)
def requires(self) -> Dict[str, luigi.Task]:
_configure_logging(self.log_dest)
summary = NAACCR_Summary(
db_url=self.db_url,
user=self.user,
passkey=self.passkey,
dateCaseReportExported=self.dateCaseReportExported,
npiRegistryId=self.npiRegistryId,
)
ont1 = NAACCR_Ontology1(
db_url=self.db_url,
user=self.user,
passkey=self.passkey,
)
return dict(NAACCR_Ontology1=ont1,
NAACCR_Summary=summary)
def run_upload(self, conn: Connection, upload_id: int) -> None:
self.run_script(
conn, self.script_name, self.script,
variables=dict(upload_id=str(upload_id),
task_id=self.task_id),
script_params=dict(
upload_id=upload_id,
task_id=self.task_id,
source_cd=self.source_cd,
update_date=self.dateCaseReportExported))
class _NAACCR_JDBC(SparkJDBCTask):
"""Load data from a NAACCR flat file into a table via JDBC.
Use a `task_id` column to manage freshness.
"""
table_name: str
dateCaseReportExported = pv.DateParam()
npiRegistryId = pv.StrParam()
def requires(self) -> Dict[str, luigi.Task]:
return {
'NAACCR_FlatFile':
NAACCR_FlatFile(dateCaseReportExported=self.dateCaseReportExported,
npiRegistryId=self.npiRegistryId)
}
def _flat_file_task(self) -> NAACCR_FlatFile:
return cast(NAACCR_FlatFile, self.requires()['NAACCR_FlatFile'])
def output(self) -> luigi.Target:
query = f"""
(select 1 from {self.table_name}
where task_id = '{self.task_id}')
"""
return JDBCTableTarget(self, query)
def main_action(self, sparkContext: SparkContext_T) -> None:
quiet_logs(sparkContext)
spark = SparkSession(sparkContext)
ff = self._flat_file_task()
naaccr_text_lines = spark.read.text(str(ff.flat_file))
data = self._data(spark, naaccr_text_lines)
# ISSUE: task_id is kinda long; how about just task_hash?
# luigi_task_hash?
data = data.withColumn('task_id', func.lit(self.task_id))
data = td.case_fold(data)
self.account().wr(data.write, self.table_name, mode='overwrite')
def _data(self, spark: SparkSession,
naaccr_text_lines: DataFrame) -> DataFrame:
raise NotImplementedError('subclass must implement')
class NAACCR_Load(_RunScriptTask):
'''Map and load NAACCR patients, tumors / visits, and facts.
'''
# flat file attributes
dateCaseReportExported = pv.DateParam()
npiRegistryId = pv.StrParam()
# encounter mapping
encounter_ide_source = pv.StrParam(default='*****@*****.**')
project_id = pv.StrParam(default='BlueHeron')
source_cd = pv.StrParam(default='*****@*****.**')
# ISSUE: task_id should depend on dest schema / owner.
z_design_id = pv.StrParam(default='nested fields')
script_name = 'naaccr_facts_load.sql'
script_deid_name = 'i2b2_facts_deid.sql'
script = res.read_text(heron_load, script_name)
script_deid = res.read_text(heron_load, script_deid_name)
@property
def classpath(self) -> str:
return self.jdbc_driver_jar
def _upload_target(self) -> 'UploadTarget':
return UploadTarget(self, self.schema, transform_name=self.task_id)
def requires(self) -> Dict[str, luigi.Task]:
_configure_logging(self.log_dest)
ff = NAACCR_FlatFile(
dateCaseReportExported=self.dateCaseReportExported,
npiRegistryId=self.npiRegistryId)
parts = {
cls.__name__:
cls(db_url=self.db_url,
user=self.user,
passkey=self.passkey,
dateCaseReportExported=self.dateCaseReportExported,
npiRegistryId=self.npiRegistryId)
for cls in [NAACCR_Patients, NAACCR_Visits, NAACCR_Facts]
}
return dict(parts, NAACCR_FlatFile=ff)
def _flat_file_task(self) -> NAACCR_FlatFile:
return cast(NAACCR_FlatFile, self.requires()['NAACCR_FlatFile'])
def _patients_task(self) -> NAACCR_Patients:
return cast(NAACCR_Patients, self.requires()['NAACCR_Patients'])
def run_upload(self, conn: Connection, upload_id: int) -> None:
ff = self._flat_file_task()
pat = self._patients_task()
# ISSUE: split these into separate tasks?
for name, script in [(self.script_name, self.script),
(self.script_deid_name, self.script_deid)]:
self.run_script(
conn,
name,
script,
variables=dict(upload_id=str(upload_id), task_id=self.task_id),
script_params=dict(
upload_id=upload_id,
project_id=self.project_id,
task_id=self.task_id,
source_cd=self.source_cd,
download_date=ff.dateCaseReportExported,
patient_ide_source=pat.patient_ide_source,
encounter_ide_source=self.encounter_ide_source))
class NAACCR_FlatFile(ManualTask):
"""A NAACCR flat file is determined by the registry, export date,
and version.
"""
naaccrRecordVersion = pv.IntParam(default=180)
dateCaseReportExported = pv.DateParam()
npiRegistryId = pv.StrParam()
testData = pv.BoolParam(default=False, significant=False)
flat_file = pv.PathParam(significant=False)
record_qty_min = pv.IntParam(significant=False, default=1)
def check_version_param(self) -> None:
"""Only version 18 (180) is currently supported.
"""
if self.naaccrRecordVersion != 180:
raise NotImplementedError()
def complete(self) -> bool:
with task_action(self, 'complete') as ctx:
result = self.complete_action()
ctx.add_success_fields(result=result)
return result
def complete_action(self) -> bool:
"""Check the first record, assuming all the others have
the same export date and registry NPI.
"""
self.check_version_param()
with self.flat_file.open() as records:
record0 = records.readline()
qty = 1 + sum(1 for _ in records.readlines())
log.info('record qty: %d (> %d? %s)', qty, self.record_qty_min,
qty >= self.record_qty_min)
vOk = self._checkItem(record0, 'naaccrRecordVersion',
str(self.naaccrRecordVersion))
regOk = self._checkItem(record0, 'npiRegistryId', self.npiRegistryId)
dtOk = self._checkItem(record0, 'dateCaseReportExported',
self.dateCaseReportExported.strftime('%Y%m%d'))
if vOk and regOk and dtOk and qty >= self.record_qty_min:
return True
else:
if self.testData:
log.warn('ignoring failed FlatFile check')
return True
return False
@classmethod
def _checkItem(cls, record: str, naaccrId: str, expected: str) -> bool:
'''
>>> npi = '1234567890'
>>> record0 = ' ' * 19 + npi
>>> NAACCR_FlatFile._checkItem(record0, 'npiRegistryId', npi)
True
>>> NAACCR_FlatFile._checkItem(record0, 'npiRegistryId', 'XXX')
False
'''
itemDef = tr_ont.NAACCR1.itemDef(naaccrId)
[startColumn, length
] = [int(itemDef.attrib[it]) for it in ['startColumn', 'length']]
startColumn -= 1
actual = record[startColumn:startColumn + length]
if actual != expected:
log.warn('%s: expected %s [%s:%s] = {%s} but found {%s}',
cls.__name__, naaccrId, startColumn - 1,
startColumn + length, expected, actual)
return actual == expected