Esempio n. 1
0
 def __indexNCleanPackages(self):
     """
     Index and clean packages - will force caching if not already in cache
     """
     logging.info("%s: Packages - packaging Packages Index ..." % self.vistaLabel)
     start = datetime.now()
     self.__noSpecificValues = 0
     # TODO: move to dict of dicts. Dynamic naming.
     self.__packageAbouts = OrderedDict()
     self.__packageVersions = {}
     self.__packageFiles = {}
     self.__filesPackage = {} # from file to Package
     self.__prefixes = defaultdict(list)
     self.__excludedPrefixes = defaultdict(list)
     limit = 1000 if self.vistaLabel == "GOLD" else VistaPackages.__ALL_LIMIT
     cstop = 10 if self.vistaLabel == "GOLD" else VistaPackages.__CSTOP
     for i, packageResult in enumerate(self.__fmqlCacher.describeFileEntries("9_4", limit=limit, cstop=cstop)):
         # logging.info("... package result %d" % i)
         dr = FMQLDescribeResult(packageResult)
         self.__noSpecificValues += dr.noSpecificValues()
         name = packageResult["name"]["value"]
         if name in self.__packageAbouts:
             raise Exception("Two packages in this VistA have the same name %s - breaks assumptions" % name)
         self.__packageAbouts[name] = dr.cstopped(flatten=True)
         self.__packageAbouts[name]["vse:ien"] = packageResult["uri"]["value"].split("-")[1]
         if "file" in dr.cnodeFields():
             # catch missing 'file'. TBD: do verify version?
             self.__packageFiles[name] = [cnode for cnode in dr.cnodes("file") if "file" in cnode]
             # turn 1- form into straight file id. Note dd_number is optional
             for fileAbout in self.__packageFiles[name]:
                 # TODO: file name - want to 
                 fileAbout["vse:file_id"] = fileAbout["file"][2:]
         if "version" in dr.cnodeFields():
             self.__packageVersions[name] = [cnode for cnode in dr.cnodes("version") if "version" in cnode]
             last = self.__packageVersions[name][-1]
             if "date_installed_at_this_site" in last:
                 self.__packageAbouts[name]["vse:last_installed"] = last["date_installed_at_this_site"]
         # Should only be one package per main
         if "prefix" in packageResult:
             if packageResult["prefix"]["value"] in self.__prefixes:
                 raise Exception("Expected main prefix to belong to only one package but at least two have it - %s in %s and %s" % (packageResult["prefix"]["value"], self.__prefixes[packageResult["prefix"]["value"]][0], name))
             self.__prefixes[packageResult["prefix"]["value"]].append((name, True))
         # ie/ to go along with 'prefix', gathered additional and excluded
         # ex/ of DPT to include File PATIENT in Registration.
         if "additional_prefixes" in dr.cnodeFields():
             for additional in [cnode["additional_prefixes"] for cnode in dr.cnodes("additional_prefixes") if "additional_prefixes" in cnode]:
                 # has to be an array ie/ same prefix -> > 1 pkg 
                 # ex/ XPD to KERNEL and KIDS
                 self.__prefixes[additional].append((name, False))
         # ex/ "PSZ" for "PS" package says "PSZ" isn't in "PS" scope.
         if "excluded_name_space" in dr.cnodeFields():
             for excluded in [cnode["excluded_name_space"] for cnode in dr.cnodes("excluded_name_space") if "excluded_name_space" in cnode]:
                 # can have > 1 PKG ex/ "ZZ" in ONCOLOGY and TOOLKIT
                 self.__excludedPrefixes[excluded].append(name)
             
     logging.info("%s: Indexing, cleaning (with caching) %d packages took %s" % (self.vistaLabel, len(self.__packageAbouts), datetime.now()-start))
Esempio n. 2
0
    def __indexNCleanBuilds(self):
        """
        Index and clean builds - will force caching if not already in cache
        
        CNodes: only see ...
        'required_build', u'install_questions', u'multiple_build', u'file', 'build_components', u'package_namespace_or_prefix' 
        but no "global"
        """
        logging.info("%s: Builds - building Builds Index ..." % self.vistaLabel)
        start = datetime.now()
        self.__noSpecificValues = 0
        # TODO: move to dict of dicts. Dynamic naming.
        self.__buildAbouts = OrderedDict()
        self.__buildFiles = {}
        self.__buildMultiples = {}
        self.__buildGlobals = {}
        self.__buildRoutines = {}  # from build components
        self.__buildRPCs = {}  # from build components
        self.__buildsByPackageName = defaultdict()
        self.__packages = {}
        limit = 1000 if self.vistaLabel == "GOLD" else VistaBuilds.__ALL_LIMIT
        for i, buildResult in enumerate(self.__fmqlCacher.describeFileEntries("9_6", limit=limit, cstop=10000)):
            # logging.info("... build result %d" % i)
            dr = FMQLDescribeResult(buildResult)
            self.__noSpecificValues += dr.noSpecificValues()
            name = buildResult["name"]["value"]
            if name in self.__buildAbouts:
                raise Exception("Two builds in this VistA have the same name %s - breaks assumptions" % name)
            # Don't show FMQL itself
            if re.match(r"CGFMQL", name):
                continue
            self.__buildAbouts[name] = dr.cstopped(flatten=True)
            if "package_file_link" in buildResult:
                packageName = buildResult["package_file_link"]["label"].split("/")[1]
                self.__buildAbouts[name]["vse:package_name"] = packageName
                self.__buildAbouts[name]["vse:package"] = buildResult["package_file_link"]["value"]
                self.__buildsByPackageName[packageName] = name
                self.__packages[buildResult["package_file_link"]["value"]] = packageName
            self.__buildAbouts[name]["vse:ien"] = buildResult["uri"]["value"].split("-")[1]
            self.__buildAbouts[name]["vse:status"] = "NEVER_INSTALLED"  # overridden below
            if "file" in dr.cnodeFields():
                # catch missing 'file'. TBD: do verify version?
                self.__buildFiles[name] = [cnode for cnode in dr.cnodes("file") if "file" in cnode]
                # turn 1- form into straight file id. Note dd_number is optional
                for fileAbout in self.__buildFiles[name]:
                    fileAbout["vse:file_id"] = fileAbout["file"][2:]
            if "global" in dr.cnodeFields():
                self.__buildGlobals[name] = [cnode for cnode in dr.cnodes("global") if "global" in cnode]
            if "multiple_build" in dr.cnodeFields():
                self.__buildMultiples[name] = [
                    cnode for cnode in dr.cnodes("multiple_build") if "multiple_build" in cnode
                ]
            # TODO: required build for tracing if want to be full Build analysis framework
            if "package_namespace_or_prefix" in dr.cnodeFields():
                pass  # may join?
            # Strange structure: entry for all possibilities but only some have data
            if "build_components" in dr.cnodeFields():
                bcs = dr.cnodes("build_components")
                for bc in bcs:
                    if "entries" not in bc:
                        continue
                    if bc["build_component"] == "1-8994":
                        self.__buildRPCs[name] = bc["entries"]
                    if bc["build_component"] == "1-9.8":
                        self.__buildRoutines[name] = bc["entries"]
                    continue
        logging.info(
            "%s: Indexing, cleaning (with caching) %d builds took %s"
            % (self.vistaLabel, len(self.__buildAbouts), datetime.now() - start)
        )
        self.__installAbouts = OrderedDict()
        noInstalls = 0
        for i, installResult in enumerate(self.__fmqlCacher.describeFileEntries("9_7", limit=limit, cstop=0)):
            # WV has entries with no status: usually there is a follow on with data
            if "status" not in installResult:
                logging.error("No 'status' in install %s" % installResult["uri"]["value"])
                continue
            ir = FMQLDescribeResult(installResult)
            self.__noSpecificValues += ir.noSpecificValues()
            name = installResult["name"]["value"]
            # Don't show FMQL itself
            if re.match(r"CGFMQL", name):
                continue
            if name not in self.__installAbouts:
                self.__installAbouts[name] = []
            self.__installAbouts[name].append(ir.cstopped(flatten=True))
            noInstalls += 1

        # Finally let's go through these installs (in order), all have status
        # and note various aspects of the build like if still installed, last install
        # time etc.
        self.__buildAboutsInstalled = OrderedDict()
        for name, installInfos in self.__installAbouts.items():
            if name not in self.__buildAbouts:
                continue  # TODO: look at this: FOIA GECS*2.0*10 (corrupt FOIA?)
            for installInfo in installInfos:
                if installInfo["status"] == "Install Completed":
                    try:
                        self.__buildAbouts[name]["vse:last_install_effect"] = installInfo["install_complete_time"]
                    except:  # TODO: check this further - 0LR*5.2*156 in VAVISTA
                        self.__buildAbouts[name]["vse:last_install_effect"] = installInfo["install_start_time"]
                    self.__buildAbouts[name]["vse:status"] = "INSTALLED"
                    if name in self.__buildAboutsInstalled:
                        del self.__buildAboutsInstalled[name]
                    self.__buildAboutsInstalled[name] = self.__buildAbouts[name]
                elif installInfo["status"] == "De-Installed":
                    # TODO: no obvious field for this.
                    self.__buildAbouts[name]["vse:last_install_effect"] = ""
                    self.__buildAbouts[name]["vse:status"] = "DE_INSTALLED"
                    # Should always be but just in case
                    if name in self.__buildAboutsInstalled:
                        del self.__buildAboutsInstalled[name]
                    else:
                        logging.error("De-installing an uninstalled build: %s" % installInfo["uri"])

        logging.info(
            "%s: Indexing, cleaning (with caching) %d builds, %d installs took %s"
            % (self.vistaLabel, len(self.__buildAbouts), noInstalls, datetime.now() - start)
        )
Esempio n. 3
0
    def __indexNCleanPackages(self):
        """
        Index and clean packages - will force caching if not already in cache
        """
        logging.info("%s: Packages - packaging Packages Index ..." %
                     self.vistaLabel)
        start = datetime.now()
        self.__noSpecificValues = 0
        # TODO: move to dict of dicts. Dynamic naming.
        self.__packageAbouts = OrderedDict()
        self.__packageVersions = {}
        self.__packageFiles = {}
        self.__filesPackage = {}  # from file to Package
        self.__prefixes = defaultdict(list)
        self.__excludedPrefixes = defaultdict(list)
        limit = 1000 if self.vistaLabel == "GOLD" else VistaPackages.__ALL_LIMIT
        cstop = 10 if self.vistaLabel == "GOLD" else VistaPackages.__CSTOP
        for i, packageResult in enumerate(
                self.__fmqlCacher.describeFileEntries("9_4",
                                                      limit=limit,
                                                      cstop=cstop)):
            # logging.info("... package result %d" % i)
            dr = FMQLDescribeResult(packageResult)
            self.__noSpecificValues += dr.noSpecificValues()
            name = packageResult["name"]["value"]
            if name in self.__packageAbouts:
                raise Exception(
                    "Two packages in this VistA have the same name %s - breaks assumptions"
                    % name)
            self.__packageAbouts[name] = dr.cstopped(flatten=True)
            self.__packageAbouts[name]["vse:ien"] = packageResult["uri"][
                "value"].split("-")[1]
            if "file" in dr.cnodeFields():
                # catch missing 'file'. TBD: do verify version?
                self.__packageFiles[name] = [
                    cnode for cnode in dr.cnodes("file") if "file" in cnode
                ]
                # turn 1- form into straight file id. Note dd_number is optional
                for fileAbout in self.__packageFiles[name]:
                    # TODO: file name - want to
                    fileAbout["vse:file_id"] = fileAbout["file"][2:]
            if "version" in dr.cnodeFields():
                self.__packageVersions[name] = [
                    cnode for cnode in dr.cnodes("version")
                    if "version" in cnode
                ]
                last = self.__packageVersions[name][-1]
                if "date_installed_at_this_site" in last:
                    self.__packageAbouts[name]["vse:last_installed"] = last[
                        "date_installed_at_this_site"]
            # Should only be one package per main
            if "prefix" in packageResult:
                if packageResult["prefix"]["value"] in self.__prefixes:
                    raise Exception(
                        "Expected main prefix to belong to only one package but at least two have it - %s in %s and %s"
                        %
                        (packageResult["prefix"]["value"],
                         self.__prefixes[packageResult["prefix"]["value"]][0],
                         name))
                self.__prefixes[packageResult["prefix"]["value"]].append(
                    (name, True))
            # ie/ to go along with 'prefix', gathered additional and excluded
            # ex/ of DPT to include File PATIENT in Registration.
            if "additional_prefixes" in dr.cnodeFields():
                for additional in [
                        cnode["additional_prefixes"]
                        for cnode in dr.cnodes("additional_prefixes")
                        if "additional_prefixes" in cnode
                ]:
                    # has to be an array ie/ same prefix -> > 1 pkg
                    # ex/ XPD to KERNEL and KIDS
                    self.__prefixes[additional].append((name, False))
            # ex/ "PSZ" for "PS" package says "PSZ" isn't in "PS" scope.
            if "excluded_name_space" in dr.cnodeFields():
                for excluded in [
                        cnode["excluded_name_space"]
                        for cnode in dr.cnodes("excluded_name_space")
                        if "excluded_name_space" in cnode
                ]:
                    # can have > 1 PKG ex/ "ZZ" in ONCOLOGY and TOOLKIT
                    self.__excludedPrefixes[excluded].append(name)

        logging.info(
            "%s: Indexing, cleaning (with caching) %d packages took %s" %
            (self.vistaLabel, len(
                self.__packageAbouts), datetime.now() - start))
Esempio n. 4
0
    def __indexNCleanBuilds(self):
        """
        Index and clean builds - will force caching if not already in cache
        
        CNodes: only see ...
        'required_build', u'install_questions', u'multiple_build', u'file', 'build_components', u'package_namespace_or_prefix' 
        but no "global"
        """
        logging.info("%s: Builds - building Builds Index ..." %
                     self.vistaLabel)
        start = datetime.now()
        self.__noSpecificValues = 0
        # TODO: move to dict of dicts. Dynamic naming.
        self.__buildAbouts = OrderedDict()
        self.__buildFiles = {}
        self.__buildMultiples = {}
        self.__buildRequired = {}
        self.__buildGlobals = {}
        self.__buildRoutines = {}  # from build components
        self.__buildRPCs = {}  # from build components
        self.__buildsByPackageName = defaultdict()
        self.__packages = {}
        limit = 1000 if self.vistaLabel == "GOLD" else VistaBuilds.__ALL_LIMIT
        for i, buildResult in enumerate(
                self.__fmqlCacher.describeFileEntries("9_6",
                                                      limit=limit,
                                                      cstop=10000)):
            # logging.info("... build result %d" % i)
            dr = FMQLDescribeResult(buildResult)
            self.__noSpecificValues += dr.noSpecificValues()
            name = buildResult["name"]["value"]
            if name in self.__buildAbouts:
                raise Exception(
                    "Two builds in this VistA have the same name %s - breaks assumptions"
                    % name)
            # Don't show FMQL itself
            if re.match(r'CGFMQL', name):
                continue
            self.__buildAbouts[name] = dr.cstopped(flatten=True)
            if "package_file_link" in buildResult:
                packageName = buildResult["package_file_link"]["label"].split(
                    "/")[1]
                self.__buildAbouts[name]["vse:package_name"] = packageName
                # VAVISTA/FMQL bug? {u'fmId': u'1', u'fmType': u'7', u'type': u'uri', u'value': u'9_4-RADIOLOGY/NUCLEAR MEDICINE', u'label': u'PACKAGE/RADIOLOGY_NUCLEAR MEDICINE'}
                packageId = buildResult["package_file_link"][
                    "value"] if not re.search(
                        r'[A-Za-z]', buildResult["package_file_link"]
                        ["value"]) else "9_4-10000"
                self.__buildAbouts[name]["vse:package"] = packageId
                self.__buildsByPackageName[packageName] = name
                self.__packages[packageId] = packageName
            self.__buildAbouts[name]["vse:ien"] = buildResult["uri"][
                "value"].split("-")[1]
            self.__buildAbouts[name][
                "vse:status"] = "NEVER_INSTALLED"  # overridden below
            if "file" in dr.cnodeFields():
                # catch missing 'file'. TBD: do verify version?
                self.__buildFiles[name] = [
                    cnode for cnode in dr.cnodes("file") if "file" in cnode
                ]
                # turn 1- form into straight file id. Note dd_number is optional
                for fileAbout in self.__buildFiles[name]:
                    fileAbout["vse:file_id"] = fileAbout["file"][2:]
            if "global" in dr.cnodeFields():
                self.__buildGlobals[name] = [
                    cnode for cnode in dr.cnodes("global") if "global" in cnode
                ]
            if "multiple_build" in dr.cnodeFields():
                self.__buildMultiples[name] = [
                    cnode for cnode in dr.cnodes("multiple_build")
                    if "multiple_build" in cnode
                ]
            if "required_build" in dr.cnodeFields():
                self.__buildRequired[name] = [
                    cnode for cnode in dr.cnodes("required_build")
                    if "required_build" in cnode
                ]
            # TODO: required build for tracing if want to be full Build analysis framework
            if "package_namespace_or_prefix" in dr.cnodeFields():
                pass  # may join?
            # Strange structure: entry for all possibilities but only some have data
            if "build_components" in dr.cnodeFields():
                bcs = dr.cnodes("build_components")
                for bc in bcs:
                    if "entries" not in bc:
                        continue
                    if bc["build_component"] == "1-8994":
                        self.__buildRPCs[name] = bc["entries"]
                    if bc["build_component"] == "1-9.8":
                        self.__buildRoutines[name] = bc["entries"]
                    continue
        logging.info(
            "%s: Indexing, cleaning (with caching) %d builds took %s" %
            (self.vistaLabel, len(self.__buildAbouts), datetime.now() - start))
        self.__installAbouts = OrderedDict()
        noInstalls = 0
        for i, installResult in enumerate(
                self.__fmqlCacher.describeFileEntries("9_7",
                                                      limit=limit,
                                                      cstop=0)):
            # WV has entries with no status: usually there is a follow on with data
            if "status" not in installResult:
                logging.error("No 'status' in install %s" %
                              installResult["uri"]["value"])
                continue
            ir = FMQLDescribeResult(installResult)
            self.__noSpecificValues += ir.noSpecificValues()
            name = installResult["name"]["value"]
            # Don't show FMQL itself
            if re.match(r'CGFMQL', name):
                continue
            if name not in self.__installAbouts:
                self.__installAbouts[name] = []
            self.__installAbouts[name].append(ir.cstopped(flatten=True))
            noInstalls += 1

        # Finally let's go through these installs (in order), all have status
        # and note various aspects of the build like if still installed, last install
        # time etc.
        self.__buildAboutsInstalled = OrderedDict()
        for name, installInfos in self.__installAbouts.items():
            if name not in self.__buildAbouts:
                continue  # TODO: look at this: FOIA GECS*2.0*10 (corrupt FOIA?)
            for installInfo in installInfos:
                if installInfo["status"] == "Install Completed":
                    try:
                        self.__buildAbouts[name][
                            "vse:last_install_effect"] = installInfo[
                                "install_complete_time"]
                    except:  # TODO: check this further - 0LR*5.2*156 in VAVISTA
                        self.__buildAbouts[name][
                            "vse:last_install_effect"] = installInfo[
                                "install_start_time"]
                    self.__buildAbouts[name]["vse:status"] = "INSTALLED"
                    if name in self.__buildAboutsInstalled:
                        del self.__buildAboutsInstalled[name]
                    self.__buildAboutsInstalled[name] = self.__buildAbouts[
                        name]
                elif installInfo["status"] == "De-Installed":
                    # TODO: no obvious field for this.
                    self.__buildAbouts[name]["vse:last_install_effect"] = ""
                    self.__buildAbouts[name]["vse:status"] = "DE_INSTALLED"
                    # Should always be but just in case
                    if name in self.__buildAboutsInstalled:
                        del self.__buildAboutsInstalled[name]
                    else:
                        logging.error(
                            "De-installing an uninstalled build: %s" %
                            installInfo["uri"])

        logging.info(
            "%s: Indexing, cleaning (with caching) %d builds, %d installs took %s"
            % (self.vistaLabel, len(
                self.__buildAbouts), noInstalls, datetime.now() - start))