def __indexNCleanBuilds(self): """ Index and clean builds - will force caching if not already in cache CNodes: only see ... 'required_build', u'install_questions', u'multiple_build', u'file', 'build_components', u'package_namespace_or_prefix' but no "global" """ logging.info("%s: Builds - building Builds Index ..." % self.vistaLabel) start = datetime.now() self.__noSpecificValues = 0 # TODO: move to dict of dicts. Dynamic naming. self.__buildAbouts = OrderedDict() self.__buildFiles = {} self.__buildMultiples = {} self.__buildGlobals = {} self.__buildRoutines = {} # from build components self.__buildRPCs = {} # from build components self.__buildsByPackageName = defaultdict() self.__packages = {} limit = 1000 if self.vistaLabel == "GOLD" else VistaBuilds.__ALL_LIMIT for i, buildResult in enumerate(self.__fmqlCacher.describeFileEntries("9_6", limit=limit, cstop=10000)): # logging.info("... build result %d" % i) dr = FMQLDescribeResult(buildResult) self.__noSpecificValues += dr.noSpecificValues() name = buildResult["name"]["value"] if name in self.__buildAbouts: raise Exception("Two builds in this VistA have the same name %s - breaks assumptions" % name) # Don't show FMQL itself if re.match(r"CGFMQL", name): continue self.__buildAbouts[name] = dr.cstopped(flatten=True) if "package_file_link" in buildResult: packageName = buildResult["package_file_link"]["label"].split("/")[1] self.__buildAbouts[name]["vse:package_name"] = packageName self.__buildAbouts[name]["vse:package"] = buildResult["package_file_link"]["value"] self.__buildsByPackageName[packageName] = name self.__packages[buildResult["package_file_link"]["value"]] = packageName self.__buildAbouts[name]["vse:ien"] = buildResult["uri"]["value"].split("-")[1] self.__buildAbouts[name]["vse:status"] = "NEVER_INSTALLED" # overridden below if "file" in dr.cnodeFields(): # catch missing 'file'. TBD: do verify version? self.__buildFiles[name] = [cnode for cnode in dr.cnodes("file") if "file" in cnode] # turn 1- form into straight file id. Note dd_number is optional for fileAbout in self.__buildFiles[name]: fileAbout["vse:file_id"] = fileAbout["file"][2:] if "global" in dr.cnodeFields(): self.__buildGlobals[name] = [cnode for cnode in dr.cnodes("global") if "global" in cnode] if "multiple_build" in dr.cnodeFields(): self.__buildMultiples[name] = [ cnode for cnode in dr.cnodes("multiple_build") if "multiple_build" in cnode ] # TODO: required build for tracing if want to be full Build analysis framework if "package_namespace_or_prefix" in dr.cnodeFields(): pass # may join? # Strange structure: entry for all possibilities but only some have data if "build_components" in dr.cnodeFields(): bcs = dr.cnodes("build_components") for bc in bcs: if "entries" not in bc: continue if bc["build_component"] == "1-8994": self.__buildRPCs[name] = bc["entries"] if bc["build_component"] == "1-9.8": self.__buildRoutines[name] = bc["entries"] continue logging.info( "%s: Indexing, cleaning (with caching) %d builds took %s" % (self.vistaLabel, len(self.__buildAbouts), datetime.now() - start) ) self.__installAbouts = OrderedDict() noInstalls = 0 for i, installResult in enumerate(self.__fmqlCacher.describeFileEntries("9_7", limit=limit, cstop=0)): # WV has entries with no status: usually there is a follow on with data if "status" not in installResult: logging.error("No 'status' in install %s" % installResult["uri"]["value"]) continue ir = FMQLDescribeResult(installResult) self.__noSpecificValues += ir.noSpecificValues() name = installResult["name"]["value"] # Don't show FMQL itself if re.match(r"CGFMQL", name): continue if name not in self.__installAbouts: self.__installAbouts[name] = [] self.__installAbouts[name].append(ir.cstopped(flatten=True)) noInstalls += 1 # Finally let's go through these installs (in order), all have status # and note various aspects of the build like if still installed, last install # time etc. self.__buildAboutsInstalled = OrderedDict() for name, installInfos in self.__installAbouts.items(): if name not in self.__buildAbouts: continue # TODO: look at this: FOIA GECS*2.0*10 (corrupt FOIA?) for installInfo in installInfos: if installInfo["status"] == "Install Completed": try: self.__buildAbouts[name]["vse:last_install_effect"] = installInfo["install_complete_time"] except: # TODO: check this further - 0LR*5.2*156 in VAVISTA self.__buildAbouts[name]["vse:last_install_effect"] = installInfo["install_start_time"] self.__buildAbouts[name]["vse:status"] = "INSTALLED" if name in self.__buildAboutsInstalled: del self.__buildAboutsInstalled[name] self.__buildAboutsInstalled[name] = self.__buildAbouts[name] elif installInfo["status"] == "De-Installed": # TODO: no obvious field for this. self.__buildAbouts[name]["vse:last_install_effect"] = "" self.__buildAbouts[name]["vse:status"] = "DE_INSTALLED" # Should always be but just in case if name in self.__buildAboutsInstalled: del self.__buildAboutsInstalled[name] else: logging.error("De-installing an uninstalled build: %s" % installInfo["uri"]) logging.info( "%s: Indexing, cleaning (with caching) %d builds, %d installs took %s" % (self.vistaLabel, len(self.__buildAbouts), noInstalls, datetime.now() - start) )
def __indexNCleanPackages(self): """ Index and clean packages - will force caching if not already in cache """ logging.info("%s: Packages - packaging Packages Index ..." % self.vistaLabel) start = datetime.now() self.__noSpecificValues = 0 # TODO: move to dict of dicts. Dynamic naming. self.__packageAbouts = OrderedDict() self.__packageVersions = {} self.__packageFiles = {} self.__filesPackage = {} # from file to Package self.__prefixes = defaultdict(list) self.__excludedPrefixes = defaultdict(list) limit = 1000 if self.vistaLabel == "GOLD" else VistaPackages.__ALL_LIMIT cstop = 10 if self.vistaLabel == "GOLD" else VistaPackages.__CSTOP for i, packageResult in enumerate(self.__fmqlCacher.describeFileEntries("9_4", limit=limit, cstop=cstop)): # logging.info("... package result %d" % i) dr = FMQLDescribeResult(packageResult) self.__noSpecificValues += dr.noSpecificValues() name = packageResult["name"]["value"] if name in self.__packageAbouts: raise Exception("Two packages in this VistA have the same name %s - breaks assumptions" % name) self.__packageAbouts[name] = dr.cstopped(flatten=True) self.__packageAbouts[name]["vse:ien"] = packageResult["uri"]["value"].split("-")[1] if "file" in dr.cnodeFields(): # catch missing 'file'. TBD: do verify version? self.__packageFiles[name] = [cnode for cnode in dr.cnodes("file") if "file" in cnode] # turn 1- form into straight file id. Note dd_number is optional for fileAbout in self.__packageFiles[name]: # TODO: file name - want to fileAbout["vse:file_id"] = fileAbout["file"][2:] if "version" in dr.cnodeFields(): self.__packageVersions[name] = [cnode for cnode in dr.cnodes("version") if "version" in cnode] last = self.__packageVersions[name][-1] if "date_installed_at_this_site" in last: self.__packageAbouts[name]["vse:last_installed"] = last["date_installed_at_this_site"] # Should only be one package per main if "prefix" in packageResult: if packageResult["prefix"]["value"] in self.__prefixes: raise Exception("Expected main prefix to belong to only one package but at least two have it - %s in %s and %s" % (packageResult["prefix"]["value"], self.__prefixes[packageResult["prefix"]["value"]][0], name)) self.__prefixes[packageResult["prefix"]["value"]].append((name, True)) # ie/ to go along with 'prefix', gathered additional and excluded # ex/ of DPT to include File PATIENT in Registration. if "additional_prefixes" in dr.cnodeFields(): for additional in [cnode["additional_prefixes"] for cnode in dr.cnodes("additional_prefixes") if "additional_prefixes" in cnode]: # has to be an array ie/ same prefix -> > 1 pkg # ex/ XPD to KERNEL and KIDS self.__prefixes[additional].append((name, False)) # ex/ "PSZ" for "PS" package says "PSZ" isn't in "PS" scope. if "excluded_name_space" in dr.cnodeFields(): for excluded in [cnode["excluded_name_space"] for cnode in dr.cnodes("excluded_name_space") if "excluded_name_space" in cnode]: # can have > 1 PKG ex/ "ZZ" in ONCOLOGY and TOOLKIT self.__excludedPrefixes[excluded].append(name) logging.info("%s: Indexing, cleaning (with caching) %d packages took %s" % (self.vistaLabel, len(self.__packageAbouts), datetime.now()-start))
def __indexNCleanPackages(self): """ Index and clean packages - will force caching if not already in cache """ logging.info("%s: Packages - packaging Packages Index ..." % self.vistaLabel) start = datetime.now() self.__noSpecificValues = 0 # TODO: move to dict of dicts. Dynamic naming. self.__packageAbouts = OrderedDict() self.__packageVersions = {} self.__packageFiles = {} self.__filesPackage = {} # from file to Package self.__prefixes = defaultdict(list) self.__excludedPrefixes = defaultdict(list) limit = 1000 if self.vistaLabel == "GOLD" else VistaPackages.__ALL_LIMIT cstop = 10 if self.vistaLabel == "GOLD" else VistaPackages.__CSTOP for i, packageResult in enumerate( self.__fmqlCacher.describeFileEntries("9_4", limit=limit, cstop=cstop)): # logging.info("... package result %d" % i) dr = FMQLDescribeResult(packageResult) self.__noSpecificValues += dr.noSpecificValues() name = packageResult["name"]["value"] if name in self.__packageAbouts: raise Exception( "Two packages in this VistA have the same name %s - breaks assumptions" % name) self.__packageAbouts[name] = dr.cstopped(flatten=True) self.__packageAbouts[name]["vse:ien"] = packageResult["uri"][ "value"].split("-")[1] if "file" in dr.cnodeFields(): # catch missing 'file'. TBD: do verify version? self.__packageFiles[name] = [ cnode for cnode in dr.cnodes("file") if "file" in cnode ] # turn 1- form into straight file id. Note dd_number is optional for fileAbout in self.__packageFiles[name]: # TODO: file name - want to fileAbout["vse:file_id"] = fileAbout["file"][2:] if "version" in dr.cnodeFields(): self.__packageVersions[name] = [ cnode for cnode in dr.cnodes("version") if "version" in cnode ] last = self.__packageVersions[name][-1] if "date_installed_at_this_site" in last: self.__packageAbouts[name]["vse:last_installed"] = last[ "date_installed_at_this_site"] # Should only be one package per main if "prefix" in packageResult: if packageResult["prefix"]["value"] in self.__prefixes: raise Exception( "Expected main prefix to belong to only one package but at least two have it - %s in %s and %s" % (packageResult["prefix"]["value"], self.__prefixes[packageResult["prefix"]["value"]][0], name)) self.__prefixes[packageResult["prefix"]["value"]].append( (name, True)) # ie/ to go along with 'prefix', gathered additional and excluded # ex/ of DPT to include File PATIENT in Registration. if "additional_prefixes" in dr.cnodeFields(): for additional in [ cnode["additional_prefixes"] for cnode in dr.cnodes("additional_prefixes") if "additional_prefixes" in cnode ]: # has to be an array ie/ same prefix -> > 1 pkg # ex/ XPD to KERNEL and KIDS self.__prefixes[additional].append((name, False)) # ex/ "PSZ" for "PS" package says "PSZ" isn't in "PS" scope. if "excluded_name_space" in dr.cnodeFields(): for excluded in [ cnode["excluded_name_space"] for cnode in dr.cnodes("excluded_name_space") if "excluded_name_space" in cnode ]: # can have > 1 PKG ex/ "ZZ" in ONCOLOGY and TOOLKIT self.__excludedPrefixes[excluded].append(name) logging.info( "%s: Indexing, cleaning (with caching) %d packages took %s" % (self.vistaLabel, len( self.__packageAbouts), datetime.now() - start))
def __indexNCleanBuilds(self): """ Index and clean builds - will force caching if not already in cache CNodes: only see ... 'required_build', u'install_questions', u'multiple_build', u'file', 'build_components', u'package_namespace_or_prefix' but no "global" """ logging.info("%s: Builds - building Builds Index ..." % self.vistaLabel) start = datetime.now() self.__noSpecificValues = 0 # TODO: move to dict of dicts. Dynamic naming. self.__buildAbouts = OrderedDict() self.__buildFiles = {} self.__buildMultiples = {} self.__buildRequired = {} self.__buildGlobals = {} self.__buildRoutines = {} # from build components self.__buildRPCs = {} # from build components self.__buildsByPackageName = defaultdict() self.__packages = {} limit = 1000 if self.vistaLabel == "GOLD" else VistaBuilds.__ALL_LIMIT for i, buildResult in enumerate( self.__fmqlCacher.describeFileEntries("9_6", limit=limit, cstop=10000)): # logging.info("... build result %d" % i) dr = FMQLDescribeResult(buildResult) self.__noSpecificValues += dr.noSpecificValues() name = buildResult["name"]["value"] if name in self.__buildAbouts: raise Exception( "Two builds in this VistA have the same name %s - breaks assumptions" % name) # Don't show FMQL itself if re.match(r'CGFMQL', name): continue self.__buildAbouts[name] = dr.cstopped(flatten=True) if "package_file_link" in buildResult: packageName = buildResult["package_file_link"]["label"].split( "/")[1] self.__buildAbouts[name]["vse:package_name"] = packageName # VAVISTA/FMQL bug? {u'fmId': u'1', u'fmType': u'7', u'type': u'uri', u'value': u'9_4-RADIOLOGY/NUCLEAR MEDICINE', u'label': u'PACKAGE/RADIOLOGY_NUCLEAR MEDICINE'} packageId = buildResult["package_file_link"][ "value"] if not re.search( r'[A-Za-z]', buildResult["package_file_link"] ["value"]) else "9_4-10000" self.__buildAbouts[name]["vse:package"] = packageId self.__buildsByPackageName[packageName] = name self.__packages[packageId] = packageName self.__buildAbouts[name]["vse:ien"] = buildResult["uri"][ "value"].split("-")[1] self.__buildAbouts[name][ "vse:status"] = "NEVER_INSTALLED" # overridden below if "file" in dr.cnodeFields(): # catch missing 'file'. TBD: do verify version? self.__buildFiles[name] = [ cnode for cnode in dr.cnodes("file") if "file" in cnode ] # turn 1- form into straight file id. Note dd_number is optional for fileAbout in self.__buildFiles[name]: fileAbout["vse:file_id"] = fileAbout["file"][2:] if "global" in dr.cnodeFields(): self.__buildGlobals[name] = [ cnode for cnode in dr.cnodes("global") if "global" in cnode ] if "multiple_build" in dr.cnodeFields(): self.__buildMultiples[name] = [ cnode for cnode in dr.cnodes("multiple_build") if "multiple_build" in cnode ] if "required_build" in dr.cnodeFields(): self.__buildRequired[name] = [ cnode for cnode in dr.cnodes("required_build") if "required_build" in cnode ] # TODO: required build for tracing if want to be full Build analysis framework if "package_namespace_or_prefix" in dr.cnodeFields(): pass # may join? # Strange structure: entry for all possibilities but only some have data if "build_components" in dr.cnodeFields(): bcs = dr.cnodes("build_components") for bc in bcs: if "entries" not in bc: continue if bc["build_component"] == "1-8994": self.__buildRPCs[name] = bc["entries"] if bc["build_component"] == "1-9.8": self.__buildRoutines[name] = bc["entries"] continue logging.info( "%s: Indexing, cleaning (with caching) %d builds took %s" % (self.vistaLabel, len(self.__buildAbouts), datetime.now() - start)) self.__installAbouts = OrderedDict() noInstalls = 0 for i, installResult in enumerate( self.__fmqlCacher.describeFileEntries("9_7", limit=limit, cstop=0)): # WV has entries with no status: usually there is a follow on with data if "status" not in installResult: logging.error("No 'status' in install %s" % installResult["uri"]["value"]) continue ir = FMQLDescribeResult(installResult) self.__noSpecificValues += ir.noSpecificValues() name = installResult["name"]["value"] # Don't show FMQL itself if re.match(r'CGFMQL', name): continue if name not in self.__installAbouts: self.__installAbouts[name] = [] self.__installAbouts[name].append(ir.cstopped(flatten=True)) noInstalls += 1 # Finally let's go through these installs (in order), all have status # and note various aspects of the build like if still installed, last install # time etc. self.__buildAboutsInstalled = OrderedDict() for name, installInfos in self.__installAbouts.items(): if name not in self.__buildAbouts: continue # TODO: look at this: FOIA GECS*2.0*10 (corrupt FOIA?) for installInfo in installInfos: if installInfo["status"] == "Install Completed": try: self.__buildAbouts[name][ "vse:last_install_effect"] = installInfo[ "install_complete_time"] except: # TODO: check this further - 0LR*5.2*156 in VAVISTA self.__buildAbouts[name][ "vse:last_install_effect"] = installInfo[ "install_start_time"] self.__buildAbouts[name]["vse:status"] = "INSTALLED" if name in self.__buildAboutsInstalled: del self.__buildAboutsInstalled[name] self.__buildAboutsInstalled[name] = self.__buildAbouts[ name] elif installInfo["status"] == "De-Installed": # TODO: no obvious field for this. self.__buildAbouts[name]["vse:last_install_effect"] = "" self.__buildAbouts[name]["vse:status"] = "DE_INSTALLED" # Should always be but just in case if name in self.__buildAboutsInstalled: del self.__buildAboutsInstalled[name] else: logging.error( "De-installing an uninstalled build: %s" % installInfo["uri"]) logging.info( "%s: Indexing, cleaning (with caching) %d builds, %d installs took %s" % (self.vistaLabel, len( self.__buildAbouts), noInstalls, datetime.now() - start))