def get_pip_freeze(path): #Executes pip freeze internally and saves the info a pip_freeze.txt file pip_freeze = freeze.freeze() pip_file = open(path + '/pip_freeze.txt', 'a') for line in pip_freeze: pip_file.write(line) pip_file.write('\n') pip_file.close()
def get_diagnostics_info(): # TODO(gsamfira): decide if we want any other kind of # diagnostics. packages = list(freeze.freeze()) return { "application": os.path.basename(main.__file__), "packages": packages, "os_info": _get_host_os_info(), "hostname": platform.node(), "ip_addresses": _get_local_ips(), }
def get_packages(self): """ Yields name and version of all installed packages using ``pip freeze`` :return: generator of (name, version) tuple """ for package in freeze.freeze(): (name, *version) = package.split("==") if version: version = version[0] yield (name, version or None)
def get_installed_packages(): import pip try: from pip._internal.operations import freeze except ImportError: # pip < 10.0 from pip.operations import freeze packages = [] for p in freeze.freeze(): packages.append(p) return packages
def _is_package_installed(self, package, version): # find env root and pacakge locations venv_root = self.context.venv.path site_packages = self.context.venv.lib_paths logging.info("Searching for packages in :{}".format(site_packages)) installed_pkgs = list(freeze.freeze(paths=site_packages)) logging.info("Installed packages: {}".format(installed_pkgs)) # Verify installed package version # Search for exact version or dev build version of current version. pkg_search_string = "{0}=={1}".format(package, version) dev_build_search_string = "{0}=={1}.dev".format(package, version) return any(p == pkg_search_string or p.startswith(dev_build_search_string) for p in installed_pkgs)
def check_requirements(test2list): actual_specs = set(str(s) for s in freeze.freeze()) given_specs = set(str(s) for s in test2list) missing = given_specs - actual_specs print("Note that even if you have all the required packages, your other packages may effect the running of code.") print("Note that if you want to see all your packages you can run makedaytics.packages_to()") print('Missing packages:') for s in missing: print(' {}'.format(s))
def get_frozen_deps() -> Iterable[str]: """Get the frozen dependencies that are locally installed. This should yield the same results as runnning 'pip freeze'. Returns ------- Iterable[str] The frozen dependencies as strings. """ return freeze.freeze()
def UserCode(self): # Retrieve all installed packages in the current Python environment try: from pip._internal.operations import freeze except ImportError: # If pip version is < 10.0 then use this command from pip.operations import freeze packages = freeze.freeze() if self.out_format == 'File': self.write_to_file(packages) else: self.write_to_ip_messageboard(packages)
def get_tf_version(): try: from pip._internal.operations import freeze except ImportError: # pip < 10.0 from pip.operations import freeze x = freeze.freeze() for p in x: pkg = p.split("==") if pkg[0] == "tensorflow": return pkg[1] raise Exception("TensorFlow package missing")
def get_pipfreeze_hash(): """Get a hash of the libraries installed in the current environment - generated by pip freeze Returns ------- """ reqs = freeze.freeze() binreqs = b"" for req in reqs: binreqs += req.encode() return sha1(binreqs).hexdigest()
def packages_to(): platform_sys = platform.system() release = platform.release() print("System Platform:", platform_sys, release) print("Packages:") x = freeze.freeze() with open("requirementscontext.txt", 'w') as f: for p in x: print(p) f.write(str(p)) f.write('\n')
def test_create_experiment(self): _, experiment_name, filename, args = get_test_experiment() experiment_project = 'create_experiment_project' experiment = create_experiment( filename, args, experiment_name, experiment_project, dependency_policy=StudioDependencyPolicy()) packages = [p for p in freeze.freeze()] self.assertTrue(experiment.key == experiment_name) self.assertTrue(experiment.filename == filename) self.assertTrue(experiment.args == args) self.assertTrue(experiment.project == experiment_project) self.assertTrue(sorted(experiment.pythonenv) == sorted(packages))
def get_installed_packages(): """Get all installed packages with `pip freeze`.""" packages = list(freeze.freeze()) packages.sort() installed = dict() for package in packages: try: name, version = package.split("==") except ValueError: name = package version = "Unknown" installed[name] = version return installed
class Datarun(SintelDocument, Status): """Datarun object. The Datarun object represents a single execution of a Pipeline over a set of Signals, within the context of an Experiment. It contains all the information about the environment and context where this execution took place, which potentially allows to later on reproduce the results in a new environment. It also contains information about whether the execution was successful or not, when it started and ended, and the number of events that were found in this experiment. """ experiment = fields.ReferenceField(Experiment, reverse_delete_rule=CASCADE) pipeline = fields.ReferenceField(Pipeline, reverse_delete_rule=CASCADE) start_time = fields.DateTimeField() end_time = fields.DateTimeField() software_versions = fields.ListField(fields.StringField()) budget_type = fields.StringField() budget_amount = fields.IntField() num_events = fields.IntField() _software_versions = list(freeze.freeze()) @property def signalruns(self): return Signalrun.find(datarun=self) def start(self): """Mark this Datarun as started on DB. The ``start_time`` will be set to ``datetime.utcnow()``, the ``status`` will be set to RUNNING and the software versions will be captured. """ self.start_time = datetime.utcnow() self.status = self.STATUS_RUNNING self.software_versions = self._software_versions self.save() def end(self, status): """Mark this Datarun as ended on DB. The ``end_time`` will be set to ``datetime.utcnow()``, the ``status`` will be set to the given value, and the ``num_events`` field will be populated with the sum of the events detected by the children Signalruns. """ self.end_time = datetime.utcnow() self.status = status self.num_events = Event.find(signalrun__in=self.signalruns).count() self.save()
def pip_freeze(): from pip._internal.operations import freeze _packages = freeze.freeze() packages = [] for _package in _packages: _package = _package.strip() if not _package: continue if _package.startswith("-e"): packages.append(["-e (#%d)" % len(packages), _package[3:]]) else: _pack, _ver = _package.split("==", 1) packages.append([_pack, _ver]) return packages
def python_packages(): """All installed Python packages. LRU cached, assuming no package installations during runtime. """ packages = [] for entry in freeze.freeze(exclude_editable=True): try: name, version = entry.split("==") except ValueError: pass else: packages.append(dict(name=name, version=version)) return packages
def get_extensions_info(): extensions_info = {} if freeze is not None: extensions = freeze.freeze(local_only=True) for ex in extensions: if ex.find('ckanext') > 0 and ex.find('#egg=') > 0 and ex.find( '@') > 0: ex = ex.split('#egg=') egg = ex[1] commit = ex[0].split('@')[-1] repository = ex[0].split('.git@')[0] for pattern in ['-e ', 'git+', '@' + commit]: repository = repository.replace(pattern, '') extensions_info[egg] = [repository, commit] return extensions_info
def pip_freeze_to_run_loc(run_loc): """ Saves current list of python packages to the run location :param run_loc: :return: """ import pip import datetime from pip._internal.operations import freeze reqs = freeze.freeze() dt = datetime.datetime.today().strftime("%Y%m%d") ofile = open(run_loc + f'requirements_{dt}.txt', 'w') ofile.write('\n'.join(reqs)) ofile.close()
def show_installed_packages(): from distutils.version import LooseVersion import pip if LooseVersion(pip.__version__) > LooseVersion("10"): from pip._internal.operations import freeze else: from pip.operations import freeze packages = [] for package in list(freeze.freeze()): val = package.split("==") name = val[0] version = val[1] packages.append((name, version)) return packages
def saveDependencies(cls, model, filename, joblib=False): # Find and export the Python version and Library version lib = [requirement for requirement in freeze(local_only=True)] dependencies = dict([(li.split('==')[0], li.split('==')[1]) for li in lib]) with open('dependencies.json', 'w') as outfile: json.dump(dependencies, outfile) if joblib: # serialize with Numpy arrays by using library joblib joblib_dump(model, filename) else: # serialize standard Python objects dump(model, open(filename, 'wb'))
def get_requirements() -> Dict[str, str]: try: from pip._internal.operations import freeze # type: ignore except ImportError: # pip < 10.0 from pip.operations import freeze # type: ignore cleared_reqs: Dict[str, str] = {} for r in freeze.freeze(): if ("-e " not in r) and ("stickybeak" not in r): name: str = r.split("==")[0] version: str = r.split("==")[1] cleared_reqs[name] = version return cleared_reqs
def update_dependencies(): package_list = [{ 'package_name': x.split('==')[0], 'import_name': x.split('==')[0], 'current_version': x.split('==')[1] } for x in freeze()] for package in package_list: pp, created = ProcessPackages.objects.get_or_create( package_name=package['package_name']) if created: pp.import_name = package['import_name'] pp.current_version = package['current_version'] pp.save() print(package, created)
def check_cellpose_installation(show_progress=True): tarball = 'cellpose-0.6.6.dev13+g316927e.tar.gz' # '316927eff7ad2201391957909a2114c68baee309' try: extract = [x for x in freeze.freeze() if x.startswith('cellpose')][0][-15:] assert extract == tarball[-15:] except: print(f'Installing cellpose. Please wait.') home_dir = Path.home() / '.deepflash2' home_dir.mkdir(exist_ok=True, parents=True) url = f'https://github.com/matjesg/deepflash2/releases/download/0.1.4/{tarball}' file = download_url(url, home_dir, show_progress=show_progress) subprocess.check_call([ sys.executable, "-m", "pip", "install", '--no-deps', file.as_posix() ])
def pip_list(pkg_name=None): """Dict of installed PIP packages with version. If 'pkg_name' is not None, will return only those packages matching the name.""" pip_regex = re.compile(r"(?P<name>.*)==(?P<version>[\w\.]+)") git_regex = re.compile(r".*@(?P<version>[\w]+)#egg=(?P<name>[\w]+)") pkg_dict = {} for _pkg in freeze.freeze(local_only=True): match = pip_regex.match(_pkg) or git_regex.match(_pkg) if match and (not pkg_name or (pkg_name and match.group('name').find(pkg_name) != -1)): pkg_dict[match.group('name')] = match.group('version') return pkg_dict
def __init__(self, run_id): super().__init__() self['run_id'] = run_id self['tracked_repositories'] = None self['source_tarball'] = None self['cmdint'] = dict() self['cmdint']['version'] = cmdint.__version__ self['cmdint']['copyright'] = cmdint.__copyright__ self['cmdint']['url'] = 'https://github.com/MIC-DKFZ/cmdint/' self['cmdint']['output'] = list() self['commands'] = [] self['environment'] = dict() self['environment']['platform'] = dict() self['environment']['platform']['system'] = platform.uname().system self['environment']['platform']['release'] = platform.uname().release self['environment']['platform']['version'] = platform.uname().version self['environment']['platform']['machine'] = platform.uname().machine self['environment']['platform'][ 'logical_cores'] = multiprocessing.cpu_count() self['environment']['platform']['memory_gb'] = virtual_memory( ).total / (1024**3) self['environment']['platform']['node'] = platform.uname().node self['environment']['platform']['ip'] = RunLog.get_local_ip() self['environment']['python'] = dict() self['environment']['python']['version'] = platform.python_version() self['environment']['python']['build'] = platform.python_build() self['environment']['python']['compiler'] = platform.python_compiler() self['environment']['python'][ 'implementation'] = platform.python_implementation() self['environment']['python']['imported_modules'] = dict() for el in sys.modules.keys(): module = sys.modules[el] if hasattr(module, '__version__') and not str( module.__name__).__contains__('.'): self['environment']['python']['imported_modules'][str( module.__name__)] = str(module.__version__) self['environment']['python']['pip_freeze'] = dict() for module in freeze.freeze(): module = module.split('==') if len(module) > 1: self['environment']['python']['pip_freeze'][ module[0]] = module[1]
def _call(self, *args, _pypads_env: LoggerEnv, _logger_call, _logger_output, **kwargs): pads = _pypads_env.pypads logger.info("Tracking execution to run with id " + pads.api.active_run().info.run_id) dependencies = DependencyTO(parent=_logger_output) try: # Execute pip freeze try: # noinspection PyProtectedMember,PyPackageRequirements from pip._internal.operations import freeze except ImportError: # pip < 10.0 # noinspection PyUnresolvedReferences,PyPackageRequirements from pip.operations import freeze dependencies.add_dependency(list(freeze.freeze())) except Exception as e: _logger_output.set_failure_state(e) finally: _logger_output.dependencies = dependencies.store()
def is_dev_mode() -> bool: """Detects if flambe was installed in editable mode. For more information: https://pip.pypa.io/en/latest/reference/pip_install/#editable-installs Returns ------- bool """ x = freeze.freeze() for pkg in x: if pkg.startswith("-e") and pkg.endswith("egg=flambe"): return True return False
def pip_freeze(): result = getattr(pip_freeze, '_result', None) if result is not None: return result # Read installed packages from pip freeze from pip._internal.operations.freeze import freeze distinfo = [] h = md5() for line in freeze(): line = line.strip().lower() if line == '': continue h.update(line.encode('utf-8')) dinfo = None mpkg = re.match(r'(.+)==(.+)', line) if mpkg: dinfo = DistInfo(name=mpkg.group(1), version=mpkg.group(2), commit=None) mgit = re.match(r'-e\sgit\+.+\@(.{8}).{32}\#egg=(\w+).*$', line) if mgit: dinfo = DistInfo(name=mgit.group(2), version=get_distribution(mgit.group(2)).version, commit=mgit.group(1)) if dinfo is not None: distinfo.append(dinfo) else: _logger.warn("Could not parse pip freeze line: %s", line) static_key = h.hexdigest()[:8] def _sort_key(di): d = get_distribution(di.name) ep = len(d.get_entry_map('nextgisweb.packages')) != 0 return (not ep, di.name) distinfo = sorted(distinfo, key=_sort_key) result = (static_key, tuple(distinfo)) setattr(pip_freeze, '_result', result) return result
def _install_packages(self, dependent_pkg_path, pkg_to_exclude): python_executable = self.context.venv.python_executable working_dir = self.context.package_root_path temp_dir = self.context.temp_path list_to_exclude = [ pkg_to_exclude, ] installed_pkgs = [ p.split('==')[0] for p in list(freeze.freeze(paths=self.context.venv.lib_paths)) if p.startswith('azure-') ] logging.info("Installed azure sdk packages:{}".format(installed_pkgs)) # Do not exclude list of packages in tools directory and so these tools packages will be reinstalled from repo branch we are testing root_path = os.path.abspath( os.path.join(dependent_pkg_path, "..", "..", "..")) tools_packages = find_tools_packages(root_path) installed_pkgs = [ req for req in installed_pkgs if req not in tools_packages ] list_to_exclude.extend(installed_pkgs) # install dev requirement but skip already installed package which is being tested or present in dev requirement filtered_dev_req_path = filter_dev_requirements( dependent_pkg_path, list_to_exclude, dependent_pkg_path) if filtered_dev_req_path: logging.info("Installing filtered dev requirements from {}".format( filtered_dev_req_path)) run_check_call( [ python_executable, "-m", "pip", "install", "-r", filtered_dev_req_path ], dependent_pkg_path, ) else: logging.info("dev requirements is not found to install") # install dependent package which is being verified run_check_call( [python_executable, "-m", "pip", "install", dependent_pkg_path], temp_dir)
def main(): log.debug('*** System Information ***') log.debug('Platform: %s' % sys.platform) log.debug('Python: %s' % str(sys.version).replace('\n', '')) log.debug('Pip: %s' % ', '.join(freeze.freeze())) log.info("Running lcmap-tap version %s" % __version__) # Create a QApplication object, necessary to manage the GUI control flow and settings app = QApplication(sys.argv) # session_id = "session_{}".format(MainControls.get_time()) control_window = MainControls() if control_window: # Enter the main event loop, begin event handling for application widgets until exit() is called sys.exit(app.exec_())
def validate_environment(output_dir: Path): original_environment_file = output_dir / 'requirements.txt' current_environment_list = [p for p in freeze.freeze()] if not original_environment_file.exists(): # original run with original_environment_file.open('w') as f: f.write('\n'.join(current_environment_list)) else: # compare with original with original_environment_file.open('r') as f: original_environment_list = [line.replace('\n', '') for line in f] current_environment = convert_pip_list_to_dict( current_environment_list) original_environment = convert_pip_list_to_dict( original_environment_list) compare_environments(current_environment, original_environment) logger.info( 'Validation of environment successful. All pip installed packages match ' 'original versions. Run can proceed.')
def run(self, options, args): format_control = index.FormatControl(set(), set()) wheel_cache = WheelCache(options.cache_dir, format_control) skip = set(stdlib_pkgs) if not options.freeze_all: skip.update(DEV_PKGS) freeze_kwargs = dict( requirement=options.requirements, find_links=options.find_links, local_only=options.local, user_only=options.user, skip_regex=options.skip_requirements_regex, isolated=options.isolated_mode, wheel_cache=wheel_cache, skip=skip, exclude_editable=options.exclude_editable) for line in freeze(**freeze_kwargs): sys.stdout.write(line + '\n')
def print_versions(output, libraries=[]): try: from pip._internal.operations import freeze except ImportError: from pip.operations import freeze if libraries: click.echo( "Warning: The libraries argument has no effect", err=True, ) deps = freeze.freeze() if output: with open(output, 'w') as f: for dep in deps: print(dep, file=f) else: for dep in deps: click.echo(dep)