Beispiel #1
0
 def add(self, import_):
     try:
         raise NotImplementedError()
     except (Exception, ) as e:
         if cpdb():
             pdb.set_trace()
         raise
Beispiel #2
0
    def _initialize(self, fnp_config):
        """--init option handling"""

        try:
            fnp_config = fnp_config or os.path.join(self.workdir,
                                                    self.FN_CONFIG)

            if os.path.isfile(fnp_config):
                print(
                    "pip-stripper configuration file exists already at  @ %s.  leaving it alone"
                    % (fnp_config))
                return fnp_config

            # load the template file
            fnp_template = os.path.join(self.DN, "templates/pip-stripper.yaml")
            with open(fnp_template) as fi:
                tmpl = fi.read()

            seed = fill_template(tmpl, self)

            with open(fnp_config, "w") as fo:
                fo.write(seed)

            print("pip-stripper configuration generated @ %s" % (fnp_config))
            return fnp_config

        except (Exception, ) as e:
            if cpdb():
                pdb.set_trace()
            raise
Beispiel #3
0
    def run(self):
        try:
            # raise NotImplementedError("%s.run(%s)" % (self, locals()))

            self.s_stdlib = self.mgr.s_stdlib

            for line in self.mgr.raw_imports:
                filebucket, packagename = self.parse(line)

                if packagename in self.s_stdlib:
                    if self.mgr.options.verbose:
                        logger.info("%s in std lib" % (packagename))
                    continue

                if packagename in self.s_untracked:
                    if self.mgr.options.verbose:
                        logger.info("%s in std lib" % (packagename))
                    continue

                self.packagetracker.add_import(packagename,
                                               filebucket.bucket.name)

        except (Exception, ) as e:
            if cpdb():
                pdb.set_trace()
            raise
Beispiel #4
0
    def __init__(self, mgr):
        try:
            self.mgr = mgr

            self.config = self.mgr.config.get(self.__class__.__name__)

            self.buckets = []
            self.di_bucket = {}

            self.s_untracked = set(self.mgr.config.get("untracked", []))

            bucketnames_tracker = self.config["buckets"]["precedence"]

            self.packagetracker = PackageBucketTracker(self,
                                                       bucketnames_tracker)

            self.workdir = self.mgr.workdir

            self.patre_splitline = re.compile(self.config["pattern_splitline"])

            for key, li_pattern in self.config["regex_dirs"].items():
                bucket = DirectoryPartitionerBucket(key)
                for pattern in li_pattern:
                    bucket.add(pattern)
                self.di_bucket[key] = bucket
                self.buckets.append(bucket)

            default_bucket_name = self.config["default_bucket"]
            self.default_bucket = self.di_bucket[default_bucket_name]

        except (Exception, ) as e:
            if cpdb():
                pdb.set_trace()
            raise
Beispiel #5
0
    def add_import(self, packagename, bucketname):
        try:
            bucketname_prev = self.di_packagename.get(packagename)
            if not bucketname_prev:
                # first time, just put it in
                self.di_packagename[packagename] = bucketname
                return

            if bucketname == bucketname_prev:
                # same as before, nothing to do
                return

            index_new = self.di_bucketindex[bucketname]
            index_old = self.di_bucketindex[bucketname_prev]

            if index_new < index_old:
                # higher precendence for new (ex:  prod beats dev)
                self.di_packagename[packagename] = bucketname

            if rpdb():
                pdb.set_trace()
        except (Exception, ) as e:
            if cpdb():
                pdb.set_trace()
            raise
Beispiel #6
0
    def process(self):
        try:
            if self.scan:
                self.scanner = Scanner(self)
                self.scanner.run()
                self.import_classifier.run()

                for name in self.import_classifier.packagetracker.di_packagename:
                    self.matcher.imp.feed(name)

                pips = self.pip_classifier = ClassifierPip(self)

                for set_ in pips.di_bucket.values():
                    [self.matcher.pip.feed(name) for name in set_]

                pips.run(self.import_classifier.packagetracker)

                # for name in self.li_pip:

                self.scanwriter.write()

            if self.options.build:
                self.builder.process()

        except (Exception, ) as e:
            if cpdb():
                pdb.set_trace()
            raise
Beispiel #7
0
    def __init__(self, mgr):
        try:
            self.mgr = mgr
            self.config = self.mgr.config.get(self.__class__.__name__)

            self.patre_splitline = re.compile(self.config["pattern_splitline"])

            self.di_bucket_association = self.config["buckets"]
            self.di_bucket = {}
            for k, v in self.di_bucket_association.items():
                self.di_bucket[k] = set()
                s_ = self.di_bucket_association[k] = set(v)
                try:
                    s_.remove("pass")
                except (KeyError, ) as e:
                    pass

            self.di_freeze = {}

            self.bucket_precedence = self.config["bucket_precedence"]

            # remove entries in lower precedence positions that
            # exist higher up
            li = []
            for bucketname in self.bucket_precedence:
                li.append(self.di_bucket_association[bucketname])
            enforce_set_precedence(li)

            self.warnings = self.config["warnings"]

        except (Exception, ) as e:
            if cpdb():
                pdb.set_trace()
            raise
Beispiel #8
0
    def __init__(self, options):

        try:
            self.options = options

            pwd = os.getcwd()
            self.workdir = self.options.workdir or pwd

            self.config = None

            fnp_config = self.options.config
            if self.options.init:
                fnp_config = self._initialize(fnp_config)

            if not fnp_config:
                for dn in [self.workdir, pwd]:
                    fnp_config = os.path.join(dn, self.FN_CONFIG)
                    try:
                        with open(fnp_config) as fi:
                            self.config = yload(fi)
                            break
                    except (IOError, ) as e:
                        pass
                else:
                    msg = "missing configuration file.  perhaps you wanted to use the --init option to create one?"
                    print(msg)
                    sys.exit(1)

            else:
                with open(fnp_config) as fi:
                    self.config = yload(fi)

            self.scan = not self.options.noscan

            #
            self.vars = dict()

            self.vars["scandir"] = self.workdir

            sectionname = "filenames"
            section = self.config["vars"][sectionname]

            for k, v in section.items():
                self.vars.update(**{"%s_%s" % (sectionname, k): v})

            self.import_classifier = ClassifierImport(self)

            self.scanwriter = ScanWriter(self)

            self.matcher = Matcher()

            self.builder = Builder(self)

        except (ValueError, ) as e:
            raise
        except (Exception, ) as e:
            if cpdb():
                pdb.set_trace()
            raise
Beispiel #9
0
    def parse_requirement_line(self, line):
        try:
            packagename, version = self.patre_splitline.split(line)
            return packagename
        except ValueError:
            raise

        except (Exception, ) as e:
            if cpdb():
                pdb.set_trace()
            raise
Beispiel #10
0
 def _get_fnp(self, subject):
     try:
         if subject == "templatedir":
             return os.path.join(dn_script, "templates")
         else:
             fn = self.config["vars"]["filenames"][subject]
         return os.path.join(self.workdir, fn)
     except (Exception, ) as e:
         if cpdb():
             pdb.set_trace()
         raise
Beispiel #11
0
 def is_match(self, filename):
     try:
         for patre in self.li_pattern:
             hit = patre.search(filename)
             if hit:
                 return True
         return False
     except (Exception, ) as e:
         if cpdb():
             pdb.set_trace()
         raise
Beispiel #12
0
    def classify_filename(self, filename_import):
        try:
            for bucket in self.buckets:
                found = bucket.is_match(filename_import)
                if found:
                    return bucket

            return self.default_bucket

        except (Exception, ) as e:
            if cpdb():
                pdb.set_trace()
            raise
Beispiel #13
0
    def write(self):
        try:

            pips_buckets = self.mgr.pip_classifier.di_bucket.copy()

            li_items = list(pips_buckets.items())

            for k, v in li_items:
                pips_buckets[k] = sorted(v)

                comment_key = k[:-1] + "_"
                comment_lookup = "comment_lookup_%s" % (k)

                pips_buckets[comment_key] = comment_lookup

            pips = dict(buckets=pips_buckets, freeze=self.mgr.all_freezes)

            warnings = sorted(self.mgr.pip_classifier.warnings)

            di = self.di = dict(
                import_="comment_lookup_imports",
                imports=self.mgr.import_classifier.packagetracker.classify(),
                pips=pips,
                aliase_="comment_lookup_aliases",
                aliases=self.mgr.aliases,
                warnings=warnings,
            )

            if self.mgr.options.verbose:
                di_debug = dict(
                    stdlib=sorted(self.mgr.s_stdlib),
                    all_imports=sorted(self.mgr.all_imports),
                    all_pips=sorted(self.mgr.all_pips),
                    unknown_pip_aliases=self.mgr.matcher._debug,
                )
                di["zzz_debug"] = di_debug

            fnp_tmp = self.mgr._get_fnp("tmp")
            with open(fnp_tmp, "w", encoding="utf-8") as fo:
                dump(self.di, fo, default_flow_style=False)

            commenter = Commenter(self.fnp_yaml_comments)

            commenter.comment(fnp_tmp, self.fnp_yaml)

            print("\n\nscan written to %s" % (self.fnp_yaml))

        except (Exception, ) as e:
            if cpdb():
                pdb.set_trace()
            raise
Beispiel #14
0
    def run(self, packagetracker):

        packagename_ = packagename = packagename_ = None
        try:

            di_packagename2pip = self.mgr.imp2pip
            all_imports = self.mgr.all_imports

            self.s_missing_imports = all_imports.copy()

            pip2imp = self.mgr.pip2imp

            for pipname in self.mgr.all_pips:

                packagename = pip2imp.get(pipname, pipname)

                found = False
                for bucketname in self.bucket_precedence:
                    if pipname in self.di_bucket_association[bucketname]:
                        self.di_bucket[bucketname].add(pipname)
                        found = True
                        try:
                            self.s_missing_imports.remove(packagename)
                        except (KeyError, ) as e:
                            pass
                        break

                if found:
                    continue

                bucketname = packagetracker.get_package(packagename)
                if bucketname:
                    self.di_bucket[bucketname].add(pipname)
                    try:
                        self.s_missing_imports.remove(packagename)
                    except (KeyError, ) as e:
                        pass
                    continue

                self.di_bucket["unknown"].add(pipname)

            for import_ in self.s_missing_imports:
                self.warnings.append("missing import:%s" % (import_))

        except (Exception, ) as e:
            if cpdb():
                ppp(dict(packagename=packagename, packagename_=packagename_))
                pdb.set_trace()
            raise
Beispiel #15
0
    def get(cls, mgr, directorypartitioner, filename):
        try:

            res = cls.di_filename.get(filename)
            if res:
                return res

            bucket = directorypartitioner.classify_filename(filename)
            res = cls.di_filename[filename] = cls(filename, bucket)
            return res

        except (Exception, ) as e:
            if cpdb():
                pdb.set_trace()
            raise
Beispiel #16
0
    def run(self):
        try:
            for taskname in self.tasknames:
                config = self.mgr.config.get("Command")["tasks"][taskname]

                command = Command(self.mgr, taskname, config)
                command.run()

            fnp_out = os.path.join(
                self.mgr.workdir,
                self.mgr.config["vars"]["filenames"]["liststdlib"])
        except (Exception, ) as e:
            if cpdb():
                pdb.set_trace()
            raise
Beispiel #17
0
    def parse_import(self, import_: str) -> str:
        try:
            import_ = import_.strip()

            if import_.startswith("from ") or import_.startswith("import "):
                _packagename = import_.split()[1]
                packagename = _packagename.split(".")[0]
                return packagename

            raise NotImplementedError("parse_import(%s)" % (locals()))

        except (Exception, ) as e:
            if cpdb():
                pdb.set_trace()
            raise
Beispiel #18
0
    def parse(self, line_in):
        """
        
        """
        # /Users/jluc/kds2/py2/bemyerp/lib/_baseutils.py:6:from functools import partial, wraps
        try:
            line = line_in.replace(self.workdir, "").rstrip()
            filename, import_ = self.patre_splitline.split(line)
            packagename = self.parse_import(import_)
            pythonfile = PythonFile.get(self, self, filename)

            return pythonfile, packagename

        except (Exception, ) as e:
            if cpdb():
                pdb.set_trace()
            raise
Beispiel #19
0
    def comment(self, fnp_in, fnp_out, mode="w"):
        try:
            with open(fnp_in) as fi, open(fnp_out, mode) as fo:
                for line in fi.readlines():
                    # pdb.set_trace()

                    hit = self.patre.search(line)
                    if hit:

                        indent = " " * (len(line) - len(line.lstrip()))
                        key = hit.groups()[0]
                        comment = self.lookup(key)
                        fo.write("\n%s#%s\n" % (indent, comment))
                    else:
                        fo.write(line)
        except (Exception, ) as e:
            if cpdb():
                pdb.set_trace()
            raise
Beispiel #20
0
    def process(self):
        try:

            with open(self.fnp_input_classifier) as fi:
                di_classifier = yload(fi)

            di_bucket = di_classifier["pips"]["buckets"]

            di_freeze = di_classifier["pips"]["freeze"]

            t_fn = self.config["t_filename_out"]
            print("\n\nbuild phase - generating requirements at:")
            for req, di in self.config["req_mapper"].items():

                # pick up naming scheme from req, if present
                t_fn = di.get("t_filename_out") or t_fn

                fn_o = sub_template(t_fn, dict(req=req))
                fnp_o = os.path.join(self.mgr.workdir, fn_o)

                requirements = []

                for bucketname in di["buckets"]:
                    for pipname in di_bucket[bucketname]:
                        freeze_line = di_freeze[pipname]
                        requirements.append(freeze_line)

                requirements.sort()

                with open(fnp_o, "w") as fo:
                    for line in requirements:
                        fo.write("%s\n" % (line))
                    fo.write("\n")

                print(" %s" % (fnp_o))

        except (Exception,) as e:
            if cpdb():
                pdb.set_trace()
            raise