def __indexNCleanPackages(self):
"""
Index and clean packages - will force caching if not already in cache
"""
logging.info("%s: Packages - packaging Packages Index ..." % self.vistaLabel)
start = datetime.now()
self.__noSpecificValues = 0
# TODO: move to dict of dicts. Dynamic naming.
self.__packageAbouts = OrderedDict()
self.__packageVersions = {}
self.__packageFiles = {}
self.__filesPackage = {} # from file to Package
self.__prefixes = defaultdict(list)
self.__excludedPrefixes = defaultdict(list)
limit = 1000 if self.vistaLabel == "GOLD" else VistaPackages.__ALL_LIMIT
cstop = 10 if self.vistaLabel == "GOLD" else VistaPackages.__CSTOP
for i, packageResult in enumerate(self.__fmqlCacher.describeFileEntries("9_4", limit=limit, cstop=cstop)):
# logging.info("... package result %d" % i)
dr = FMQLDescribeResult(packageResult)
self.__noSpecificValues += dr.noSpecificValues()
name = packageResult["name"]["value"]
if name in self.__packageAbouts:
raise Exception("Two packages in this VistA have the same name %s - breaks assumptions" % name)
self.__packageAbouts[name] = dr.cstopped(flatten=True)
self.__packageAbouts[name]["vse:ien"] = packageResult["uri"]["value"].split("-")[1]
if "file" in dr.cnodeFields():
# catch missing 'file'. TBD: do verify version?
self.__packageFiles[name] = [cnode for cnode in dr.cnodes("file") if "file" in cnode]
# turn 1- form into straight file id. Note dd_number is optional
for fileAbout in self.__packageFiles[name]:
# TODO: file name - want to
fileAbout["vse:file_id"] = fileAbout["file"][2:]
if "version" in dr.cnodeFields():
self.__packageVersions[name] = [cnode for cnode in dr.cnodes("version") if "version" in cnode]
last = self.__packageVersions[name][-1]
if "date_installed_at_this_site" in last:
self.__packageAbouts[name]["vse:last_installed"] = last["date_installed_at_this_site"]
# Should only be one package per main
if "prefix" in packageResult:
if packageResult["prefix"]["value"] in self.__prefixes:
raise Exception("Expected main prefix to belong to only one package but at least two have it - %s in %s and %s" % (packageResult["prefix"]["value"], self.__prefixes[packageResult["prefix"]["value"]][0], name))
self.__prefixes[packageResult["prefix"]["value"]].append((name, True))
# ie/ to go along with 'prefix', gathered additional and excluded
# ex/ of DPT to include File PATIENT in Registration.
if "additional_prefixes" in dr.cnodeFields():
for additional in [cnode["additional_prefixes"] for cnode in dr.cnodes("additional_prefixes") if "additional_prefixes" in cnode]:
# has to be an array ie/ same prefix -> > 1 pkg
# ex/ XPD to KERNEL and KIDS
self.__prefixes[additional].append((name, False))
# ex/ "PSZ" for "PS" package says "PSZ" isn't in "PS" scope.
if "excluded_name_space" in dr.cnodeFields():
for excluded in [cnode["excluded_name_space"] for cnode in dr.cnodes("excluded_name_space") if "excluded_name_space" in cnode]:
# can have > 1 PKG ex/ "ZZ" in ONCOLOGY and TOOLKIT
self.__excludedPrefixes[excluded].append(name)
logging.info("%s: Indexing, cleaning (with caching) %d packages took %s" % (self.vistaLabel, len(self.__packageAbouts), datetime.now()-start))
def __indexNCleanBuilds(self):
"""
Index and clean builds - will force caching if not already in cache
CNodes: only see ...
'required_build', u'install_questions', u'multiple_build', u'file', 'build_components', u'package_namespace_or_prefix'
but no "global"
"""
logging.info("%s: Builds - building Builds Index ..." % self.vistaLabel)
start = datetime.now()
self.__noSpecificValues = 0
# TODO: move to dict of dicts. Dynamic naming.
self.__buildAbouts = OrderedDict()
self.__buildFiles = {}
self.__buildMultiples = {}
self.__buildGlobals = {}
self.__buildRoutines = {} # from build components
self.__buildRPCs = {} # from build components
self.__buildsByPackageName = defaultdict()
self.__packages = {}
limit = 1000 if self.vistaLabel == "GOLD" else VistaBuilds.__ALL_LIMIT
for i, buildResult in enumerate(self.__fmqlCacher.describeFileEntries("9_6", limit=limit, cstop=10000)):
# logging.info("... build result %d" % i)
dr = FMQLDescribeResult(buildResult)
self.__noSpecificValues += dr.noSpecificValues()
name = buildResult["name"]["value"]
if name in self.__buildAbouts:
raise Exception("Two builds in this VistA have the same name %s - breaks assumptions" % name)
# Don't show FMQL itself
if re.match(r"CGFMQL", name):
continue
self.__buildAbouts[name] = dr.cstopped(flatten=True)
if "package_file_link" in buildResult:
packageName = buildResult["package_file_link"]["label"].split("/")[1]
self.__buildAbouts[name]["vse:package_name"] = packageName
self.__buildAbouts[name]["vse:package"] = buildResult["package_file_link"]["value"]
self.__buildsByPackageName[packageName] = name
self.__packages[buildResult["package_file_link"]["value"]] = packageName
self.__buildAbouts[name]["vse:ien"] = buildResult["uri"]["value"].split("-")[1]
self.__buildAbouts[name]["vse:status"] = "NEVER_INSTALLED" # overridden below
if "file" in dr.cnodeFields():
# catch missing 'file'. TBD: do verify version?
self.__buildFiles[name] = [cnode for cnode in dr.cnodes("file") if "file" in cnode]
# turn 1- form into straight file id. Note dd_number is optional
for fileAbout in self.__buildFiles[name]:
fileAbout["vse:file_id"] = fileAbout["file"][2:]
if "global" in dr.cnodeFields():
self.__buildGlobals[name] = [cnode for cnode in dr.cnodes("global") if "global" in cnode]
if "multiple_build" in dr.cnodeFields():
self.__buildMultiples[name] = [
cnode for cnode in dr.cnodes("multiple_build") if "multiple_build" in cnode
]
# TODO: required build for tracing if want to be full Build analysis framework
if "package_namespace_or_prefix" in dr.cnodeFields():
pass # may join?
# Strange structure: entry for all possibilities but only some have data
if "build_components" in dr.cnodeFields():
bcs = dr.cnodes("build_components")
for bc in bcs:
if "entries" not in bc:
continue
if bc["build_component"] == "1-8994":
self.__buildRPCs[name] = bc["entries"]
if bc["build_component"] == "1-9.8":
self.__buildRoutines[name] = bc["entries"]
continue
logging.info(
"%s: Indexing, cleaning (with caching) %d builds took %s"
% (self.vistaLabel, len(self.__buildAbouts), datetime.now() - start)
)
self.__installAbouts = OrderedDict()
noInstalls = 0
for i, installResult in enumerate(self.__fmqlCacher.describeFileEntries("9_7", limit=limit, cstop=0)):
# WV has entries with no status: usually there is a follow on with data
if "status" not in installResult:
logging.error("No 'status' in install %s" % installResult["uri"]["value"])
continue
ir = FMQLDescribeResult(installResult)
self.__noSpecificValues += ir.noSpecificValues()
name = installResult["name"]["value"]
# Don't show FMQL itself
if re.match(r"CGFMQL", name):
continue
if name not in self.__installAbouts:
self.__installAbouts[name] = []
self.__installAbouts[name].append(ir.cstopped(flatten=True))
noInstalls += 1
# Finally let's go through these installs (in order), all have status
# and note various aspects of the build like if still installed, last install
# time etc.
self.__buildAboutsInstalled = OrderedDict()
for name, installInfos in self.__installAbouts.items():
if name not in self.__buildAbouts:
continue # TODO: look at this: FOIA GECS*2.0*10 (corrupt FOIA?)
for installInfo in installInfos:
if installInfo["status"] == "Install Completed":
try:
self.__buildAbouts[name]["vse:last_install_effect"] = installInfo["install_complete_time"]
except: # TODO: check this further - 0LR*5.2*156 in VAVISTA
self.__buildAbouts[name]["vse:last_install_effect"] = installInfo["install_start_time"]
self.__buildAbouts[name]["vse:status"] = "INSTALLED"
if name in self.__buildAboutsInstalled:
del self.__buildAboutsInstalled[name]
self.__buildAboutsInstalled[name] = self.__buildAbouts[name]
elif installInfo["status"] == "De-Installed":
# TODO: no obvious field for this.
self.__buildAbouts[name]["vse:last_install_effect"] = ""
self.__buildAbouts[name]["vse:status"] = "DE_INSTALLED"
# Should always be but just in case
if name in self.__buildAboutsInstalled:
del self.__buildAboutsInstalled[name]
else:
logging.error("De-installing an uninstalled build: %s" % installInfo["uri"])
logging.info(
"%s: Indexing, cleaning (with caching) %d builds, %d installs took %s"
% (self.vistaLabel, len(self.__buildAbouts), noInstalls, datetime.now() - start)
)
def __indexNCleanPackages(self):
"""
Index and clean packages - will force caching if not already in cache
"""
logging.info("%s: Packages - packaging Packages Index ..." %
self.vistaLabel)
start = datetime.now()
self.__noSpecificValues = 0
# TODO: move to dict of dicts. Dynamic naming.
self.__packageAbouts = OrderedDict()
self.__packageVersions = {}
self.__packageFiles = {}
self.__filesPackage = {} # from file to Package
self.__prefixes = defaultdict(list)
self.__excludedPrefixes = defaultdict(list)
limit = 1000 if self.vistaLabel == "GOLD" else VistaPackages.__ALL_LIMIT
cstop = 10 if self.vistaLabel == "GOLD" else VistaPackages.__CSTOP
for i, packageResult in enumerate(
self.__fmqlCacher.describeFileEntries("9_4",
limit=limit,
cstop=cstop)):
# logging.info("... package result %d" % i)
dr = FMQLDescribeResult(packageResult)
self.__noSpecificValues += dr.noSpecificValues()
name = packageResult["name"]["value"]
if name in self.__packageAbouts:
raise Exception(
"Two packages in this VistA have the same name %s - breaks assumptions"
% name)
self.__packageAbouts[name] = dr.cstopped(flatten=True)
self.__packageAbouts[name]["vse:ien"] = packageResult["uri"][
"value"].split("-")[1]
if "file" in dr.cnodeFields():
# catch missing 'file'. TBD: do verify version?
self.__packageFiles[name] = [
cnode for cnode in dr.cnodes("file") if "file" in cnode
]
# turn 1- form into straight file id. Note dd_number is optional
for fileAbout in self.__packageFiles[name]:
# TODO: file name - want to
fileAbout["vse:file_id"] = fileAbout["file"][2:]
if "version" in dr.cnodeFields():
self.__packageVersions[name] = [
cnode for cnode in dr.cnodes("version")
if "version" in cnode
]
last = self.__packageVersions[name][-1]
if "date_installed_at_this_site" in last:
self.__packageAbouts[name]["vse:last_installed"] = last[
"date_installed_at_this_site"]
# Should only be one package per main
if "prefix" in packageResult:
if packageResult["prefix"]["value"] in self.__prefixes:
raise Exception(
"Expected main prefix to belong to only one package but at least two have it - %s in %s and %s"
%
(packageResult["prefix"]["value"],
self.__prefixes[packageResult["prefix"]["value"]][0],
name))
self.__prefixes[packageResult["prefix"]["value"]].append(
(name, True))
# ie/ to go along with 'prefix', gathered additional and excluded
# ex/ of DPT to include File PATIENT in Registration.
if "additional_prefixes" in dr.cnodeFields():
for additional in [
cnode["additional_prefixes"]
for cnode in dr.cnodes("additional_prefixes")
if "additional_prefixes" in cnode
]:
# has to be an array ie/ same prefix -> > 1 pkg
# ex/ XPD to KERNEL and KIDS
self.__prefixes[additional].append((name, False))
# ex/ "PSZ" for "PS" package says "PSZ" isn't in "PS" scope.
if "excluded_name_space" in dr.cnodeFields():
for excluded in [
cnode["excluded_name_space"]
for cnode in dr.cnodes("excluded_name_space")
if "excluded_name_space" in cnode
]:
# can have > 1 PKG ex/ "ZZ" in ONCOLOGY and TOOLKIT
self.__excludedPrefixes[excluded].append(name)
logging.info(
"%s: Indexing, cleaning (with caching) %d packages took %s" %
(self.vistaLabel, len(
self.__packageAbouts), datetime.now() - start))
def __indexNCleanBuilds(self):
"""
Index and clean builds - will force caching if not already in cache
CNodes: only see ...
'required_build', u'install_questions', u'multiple_build', u'file', 'build_components', u'package_namespace_or_prefix'
but no "global"
"""
logging.info("%s: Builds - building Builds Index ..." %
self.vistaLabel)
start = datetime.now()
self.__noSpecificValues = 0
# TODO: move to dict of dicts. Dynamic naming.
self.__buildAbouts = OrderedDict()
self.__buildFiles = {}
self.__buildMultiples = {}
self.__buildRequired = {}
self.__buildGlobals = {}
self.__buildRoutines = {} # from build components
self.__buildRPCs = {} # from build components
self.__buildsByPackageName = defaultdict()
self.__packages = {}
limit = 1000 if self.vistaLabel == "GOLD" else VistaBuilds.__ALL_LIMIT
for i, buildResult in enumerate(
self.__fmqlCacher.describeFileEntries("9_6",
limit=limit,
cstop=10000)):
# logging.info("... build result %d" % i)
dr = FMQLDescribeResult(buildResult)
self.__noSpecificValues += dr.noSpecificValues()
name = buildResult["name"]["value"]
if name in self.__buildAbouts:
raise Exception(
"Two builds in this VistA have the same name %s - breaks assumptions"
% name)
# Don't show FMQL itself
if re.match(r'CGFMQL', name):
continue
self.__buildAbouts[name] = dr.cstopped(flatten=True)
if "package_file_link" in buildResult:
packageName = buildResult["package_file_link"]["label"].split(
"/")[1]
self.__buildAbouts[name]["vse:package_name"] = packageName
# VAVISTA/FMQL bug? {u'fmId': u'1', u'fmType': u'7', u'type': u'uri', u'value': u'9_4-RADIOLOGY/NUCLEAR MEDICINE', u'label': u'PACKAGE/RADIOLOGY_NUCLEAR MEDICINE'}
packageId = buildResult["package_file_link"][
"value"] if not re.search(
r'[A-Za-z]', buildResult["package_file_link"]
["value"]) else "9_4-10000"
self.__buildAbouts[name]["vse:package"] = packageId
self.__buildsByPackageName[packageName] = name
self.__packages[packageId] = packageName
self.__buildAbouts[name]["vse:ien"] = buildResult["uri"][
"value"].split("-")[1]
self.__buildAbouts[name][
"vse:status"] = "NEVER_INSTALLED" # overridden below
if "file" in dr.cnodeFields():
# catch missing 'file'. TBD: do verify version?
self.__buildFiles[name] = [
cnode for cnode in dr.cnodes("file") if "file" in cnode
]
# turn 1- form into straight file id. Note dd_number is optional
for fileAbout in self.__buildFiles[name]:
fileAbout["vse:file_id"] = fileAbout["file"][2:]
if "global" in dr.cnodeFields():
self.__buildGlobals[name] = [
cnode for cnode in dr.cnodes("global") if "global" in cnode
]
if "multiple_build" in dr.cnodeFields():
self.__buildMultiples[name] = [
cnode for cnode in dr.cnodes("multiple_build")
if "multiple_build" in cnode
]
if "required_build" in dr.cnodeFields():
self.__buildRequired[name] = [
cnode for cnode in dr.cnodes("required_build")
if "required_build" in cnode
]
# TODO: required build for tracing if want to be full Build analysis framework
if "package_namespace_or_prefix" in dr.cnodeFields():
pass # may join?
# Strange structure: entry for all possibilities but only some have data
if "build_components" in dr.cnodeFields():
bcs = dr.cnodes("build_components")
for bc in bcs:
if "entries" not in bc:
continue
if bc["build_component"] == "1-8994":
self.__buildRPCs[name] = bc["entries"]
if bc["build_component"] == "1-9.8":
self.__buildRoutines[name] = bc["entries"]
continue
logging.info(
"%s: Indexing, cleaning (with caching) %d builds took %s" %
(self.vistaLabel, len(self.__buildAbouts), datetime.now() - start))
self.__installAbouts = OrderedDict()
noInstalls = 0
for i, installResult in enumerate(
self.__fmqlCacher.describeFileEntries("9_7",
limit=limit,
cstop=0)):
# WV has entries with no status: usually there is a follow on with data
if "status" not in installResult:
logging.error("No 'status' in install %s" %
installResult["uri"]["value"])
continue
ir = FMQLDescribeResult(installResult)
self.__noSpecificValues += ir.noSpecificValues()
name = installResult["name"]["value"]
# Don't show FMQL itself
if re.match(r'CGFMQL', name):
continue
if name not in self.__installAbouts:
self.__installAbouts[name] = []
self.__installAbouts[name].append(ir.cstopped(flatten=True))
noInstalls += 1
# Finally let's go through these installs (in order), all have status
# and note various aspects of the build like if still installed, last install
# time etc.
self.__buildAboutsInstalled = OrderedDict()
for name, installInfos in self.__installAbouts.items():
if name not in self.__buildAbouts:
continue # TODO: look at this: FOIA GECS*2.0*10 (corrupt FOIA?)
for installInfo in installInfos:
if installInfo["status"] == "Install Completed":
try:
self.__buildAbouts[name][
"vse:last_install_effect"] = installInfo[
"install_complete_time"]
except: # TODO: check this further - 0LR*5.2*156 in VAVISTA
self.__buildAbouts[name][
"vse:last_install_effect"] = installInfo[
"install_start_time"]
self.__buildAbouts[name]["vse:status"] = "INSTALLED"
if name in self.__buildAboutsInstalled:
del self.__buildAboutsInstalled[name]
self.__buildAboutsInstalled[name] = self.__buildAbouts[
name]
elif installInfo["status"] == "De-Installed":
# TODO: no obvious field for this.
self.__buildAbouts[name]["vse:last_install_effect"] = ""
self.__buildAbouts[name]["vse:status"] = "DE_INSTALLED"
# Should always be but just in case
if name in self.__buildAboutsInstalled:
del self.__buildAboutsInstalled[name]
else:
logging.error(
"De-installing an uninstalled build: %s" %
installInfo["uri"])
logging.info(
"%s: Indexing, cleaning (with caching) %d builds, %d installs took %s"
% (self.vistaLabel, len(
self.__buildAbouts), noInstalls, datetime.now() - start))