def get_pip_freeze(path):
#Executes pip freeze internally and saves the info a pip_freeze.txt file
pip_freeze = freeze.freeze()
pip_file = open(path + '/pip_freeze.txt', 'a')
for line in pip_freeze:
pip_file.write(line)
pip_file.write('\n')
pip_file.close()
def get_diagnostics_info():
# TODO(gsamfira): decide if we want any other kind of
# diagnostics.
packages = list(freeze.freeze())
return {
"application": os.path.basename(main.__file__),
"packages": packages,
"os_info": _get_host_os_info(),
"hostname": platform.node(),
"ip_addresses": _get_local_ips(),
}
def get_packages(self):
"""
Yields name and version of all installed packages using ``pip freeze``
:return: generator of (name, version) tuple
"""
for package in freeze.freeze():
(name, *version) = package.split("==")
if version:
version = version[0]
yield (name, version or None)
def get_installed_packages():
import pip
try:
from pip._internal.operations import freeze
except ImportError: # pip < 10.0
from pip.operations import freeze
packages = []
for p in freeze.freeze():
packages.append(p)
return packages
def _is_package_installed(self, package, version):
# find env root and pacakge locations
venv_root = self.context.venv.path
site_packages = self.context.venv.lib_paths
logging.info("Searching for packages in :{}".format(site_packages))
installed_pkgs = list(freeze.freeze(paths=site_packages))
logging.info("Installed packages: {}".format(installed_pkgs))
# Verify installed package version
# Search for exact version or dev build version of current version.
pkg_search_string = "{0}=={1}".format(package, version)
dev_build_search_string = "{0}=={1}.dev".format(package, version)
return any(p == pkg_search_string or p.startswith(dev_build_search_string) for p in installed_pkgs)
def check_requirements(test2list):
actual_specs = set(str(s) for s in freeze.freeze())
given_specs = set(str(s) for s in test2list)
missing = given_specs - actual_specs
print("Note that even if you have all the required packages, your other packages may effect the running of code.")
print("Note that if you want to see all your packages you can run makedaytics.packages_to()")
print('Missing packages:')
for s in missing:
print(' {}'.format(s))
def get_frozen_deps() -> Iterable[str]:
"""Get the frozen dependencies that are locally installed.
This should yield the same results as runnning 'pip freeze'.
Returns
-------
Iterable[str]
The frozen dependencies as strings.
"""
return freeze.freeze()
def UserCode(self):
# Retrieve all installed packages in the current Python environment
try:
from pip._internal.operations import freeze
except ImportError: # If pip version is < 10.0 then use this command
from pip.operations import freeze
packages = freeze.freeze()
if self.out_format == 'File':
self.write_to_file(packages)
else:
self.write_to_ip_messageboard(packages)
def get_tf_version():
try:
from pip._internal.operations import freeze
except ImportError: # pip < 10.0
from pip.operations import freeze
x = freeze.freeze()
for p in x:
pkg = p.split("==")
if pkg[0] == "tensorflow":
return pkg[1]
raise Exception("TensorFlow package missing")
def get_pipfreeze_hash():
"""Get a hash of the libraries installed in the current environment - generated by pip freeze
Returns
-------
"""
reqs = freeze.freeze()
binreqs = b""
for req in reqs:
binreqs += req.encode()
return sha1(binreqs).hexdigest()
def packages_to():
platform_sys = platform.system()
release = platform.release()
print("System Platform:", platform_sys, release)
print("Packages:")
x = freeze.freeze()
with open("requirementscontext.txt", 'w') as f:
for p in x:
print(p)
f.write(str(p))
f.write('\n')
def test_create_experiment(self):
_, experiment_name, filename, args = get_test_experiment()
experiment_project = 'create_experiment_project'
experiment = create_experiment(
filename, args, experiment_name, experiment_project, dependency_policy=StudioDependencyPolicy())
packages = [p for p in freeze.freeze()]
self.assertTrue(experiment.key == experiment_name)
self.assertTrue(experiment.filename == filename)
self.assertTrue(experiment.args == args)
self.assertTrue(experiment.project == experiment_project)
self.assertTrue(sorted(experiment.pythonenv) == sorted(packages))
def get_installed_packages():
"""Get all installed packages with `pip freeze`."""
packages = list(freeze.freeze())
packages.sort()
installed = dict()
for package in packages:
try:
name, version = package.split("==")
except ValueError:
name = package
version = "Unknown"
installed[name] = version
return installed
class Datarun(SintelDocument, Status):
"""Datarun object.
The Datarun object represents a single execution of a Pipeline over a set
of Signals, within the context of an Experiment.
It contains all the information about the environment and context where this
execution took place, which potentially allows to later on reproduce the results
in a new environment.
It also contains information about whether the execution was successful or not,
when it started and ended, and the number of events that were found in this experiment.
"""
experiment = fields.ReferenceField(Experiment, reverse_delete_rule=CASCADE)
pipeline = fields.ReferenceField(Pipeline, reverse_delete_rule=CASCADE)
start_time = fields.DateTimeField()
end_time = fields.DateTimeField()
software_versions = fields.ListField(fields.StringField())
budget_type = fields.StringField()
budget_amount = fields.IntField()
num_events = fields.IntField()
_software_versions = list(freeze.freeze())
@property
def signalruns(self):
return Signalrun.find(datarun=self)
def start(self):
"""Mark this Datarun as started on DB.
The ``start_time`` will be set to ``datetime.utcnow()``,
the ``status`` will be set to RUNNING and the software
versions will be captured.
"""
self.start_time = datetime.utcnow()
self.status = self.STATUS_RUNNING
self.software_versions = self._software_versions
self.save()
def end(self, status):
"""Mark this Datarun as ended on DB.
The ``end_time`` will be set to ``datetime.utcnow()``, the ``status``
will be set to the given value, and the ``num_events`` field will be
populated with the sum of the events detected by the children Signalruns.
"""
self.end_time = datetime.utcnow()
self.status = status
self.num_events = Event.find(signalrun__in=self.signalruns).count()
self.save()
def pip_freeze():
from pip._internal.operations import freeze
_packages = freeze.freeze()
packages = []
for _package in _packages:
_package = _package.strip()
if not _package:
continue
if _package.startswith("-e"):
packages.append(["-e (#%d)" % len(packages), _package[3:]])
else:
_pack, _ver = _package.split("==", 1)
packages.append([_pack, _ver])
return packages
def python_packages():
"""All installed Python packages.
LRU cached, assuming no package installations during runtime.
"""
packages = []
for entry in freeze.freeze(exclude_editable=True):
try:
name, version = entry.split("==")
except ValueError:
pass
else:
packages.append(dict(name=name, version=version))
return packages
def get_extensions_info():
extensions_info = {}
if freeze is not None:
extensions = freeze.freeze(local_only=True)
for ex in extensions:
if ex.find('ckanext') > 0 and ex.find('#egg=') > 0 and ex.find(
'@') > 0:
ex = ex.split('#egg=')
egg = ex[1]
commit = ex[0].split('@')[-1]
repository = ex[0].split('.git@')[0]
for pattern in ['-e ', 'git+', '@' + commit]:
repository = repository.replace(pattern, '')
extensions_info[egg] = [repository, commit]
return extensions_info
def pip_freeze_to_run_loc(run_loc):
"""
Saves current list of python packages to the run location
:param run_loc:
:return:
"""
import pip
import datetime
from pip._internal.operations import freeze
reqs = freeze.freeze()
dt = datetime.datetime.today().strftime("%Y%m%d")
ofile = open(run_loc + f'requirements_{dt}.txt', 'w')
ofile.write('\n'.join(reqs))
ofile.close()
def show_installed_packages():
from distutils.version import LooseVersion
import pip
if LooseVersion(pip.__version__) > LooseVersion("10"):
from pip._internal.operations import freeze
else:
from pip.operations import freeze
packages = []
for package in list(freeze.freeze()):
val = package.split("==")
name = val[0]
version = val[1]
packages.append((name, version))
return packages
def saveDependencies(cls, model, filename, joblib=False):
# Find and export the Python version and Library version
lib = [requirement for requirement in freeze(local_only=True)]
dependencies = dict([(li.split('==')[0], li.split('==')[1])
for li in lib])
with open('dependencies.json', 'w') as outfile:
json.dump(dependencies, outfile)
if joblib:
# serialize with Numpy arrays by using library joblib
joblib_dump(model, filename)
else:
# serialize standard Python objects
dump(model, open(filename, 'wb'))
def get_requirements() -> Dict[str, str]:
try:
from pip._internal.operations import freeze # type: ignore
except ImportError: # pip < 10.0
from pip.operations import freeze # type: ignore
cleared_reqs: Dict[str, str] = {}
for r in freeze.freeze():
if ("-e " not in r) and ("stickybeak" not in r):
name: str = r.split("==")[0]
version: str = r.split("==")[1]
cleared_reqs[name] = version
return cleared_reqs
def update_dependencies():
package_list = [{
'package_name': x.split('==')[0],
'import_name': x.split('==')[0],
'current_version': x.split('==')[1]
} for x in freeze()]
for package in package_list:
pp, created = ProcessPackages.objects.get_or_create(
package_name=package['package_name'])
if created:
pp.import_name = package['import_name']
pp.current_version = package['current_version']
pp.save()
print(package, created)
def check_cellpose_installation(show_progress=True):
tarball = 'cellpose-0.6.6.dev13+g316927e.tar.gz' # '316927eff7ad2201391957909a2114c68baee309'
try:
extract = [x for x in freeze.freeze()
if x.startswith('cellpose')][0][-15:]
assert extract == tarball[-15:]
except:
print(f'Installing cellpose. Please wait.')
home_dir = Path.home() / '.deepflash2'
home_dir.mkdir(exist_ok=True, parents=True)
url = f'https://github.com/matjesg/deepflash2/releases/download/0.1.4/{tarball}'
file = download_url(url, home_dir, show_progress=show_progress)
subprocess.check_call([
sys.executable, "-m", "pip", "install", '--no-deps',
file.as_posix()
])
def pip_list(pkg_name=None):
"""Dict of installed PIP packages with version.
If 'pkg_name' is not None, will return only those packages matching the
name."""
pip_regex = re.compile(r"(?P<name>.*)==(?P<version>[\w\.]+)")
git_regex = re.compile(r".*@(?P<version>[\w]+)#egg=(?P<name>[\w]+)")
pkg_dict = {}
for _pkg in freeze.freeze(local_only=True):
match = pip_regex.match(_pkg) or git_regex.match(_pkg)
if match and (not pkg_name or
(pkg_name and match.group('name').find(pkg_name) != -1)):
pkg_dict[match.group('name')] = match.group('version')
return pkg_dict
def __init__(self, run_id):
super().__init__()
self['run_id'] = run_id
self['tracked_repositories'] = None
self['source_tarball'] = None
self['cmdint'] = dict()
self['cmdint']['version'] = cmdint.__version__
self['cmdint']['copyright'] = cmdint.__copyright__
self['cmdint']['url'] = 'https://github.com/MIC-DKFZ/cmdint/'
self['cmdint']['output'] = list()
self['commands'] = []
self['environment'] = dict()
self['environment']['platform'] = dict()
self['environment']['platform']['system'] = platform.uname().system
self['environment']['platform']['release'] = platform.uname().release
self['environment']['platform']['version'] = platform.uname().version
self['environment']['platform']['machine'] = platform.uname().machine
self['environment']['platform'][
'logical_cores'] = multiprocessing.cpu_count()
self['environment']['platform']['memory_gb'] = virtual_memory(
).total / (1024**3)
self['environment']['platform']['node'] = platform.uname().node
self['environment']['platform']['ip'] = RunLog.get_local_ip()
self['environment']['python'] = dict()
self['environment']['python']['version'] = platform.python_version()
self['environment']['python']['build'] = platform.python_build()
self['environment']['python']['compiler'] = platform.python_compiler()
self['environment']['python'][
'implementation'] = platform.python_implementation()
self['environment']['python']['imported_modules'] = dict()
for el in sys.modules.keys():
module = sys.modules[el]
if hasattr(module, '__version__') and not str(
module.__name__).__contains__('.'):
self['environment']['python']['imported_modules'][str(
module.__name__)] = str(module.__version__)
self['environment']['python']['pip_freeze'] = dict()
for module in freeze.freeze():
module = module.split('==')
if len(module) > 1:
self['environment']['python']['pip_freeze'][
module[0]] = module[1]
def _call(self, *args, _pypads_env: LoggerEnv, _logger_call, _logger_output, **kwargs):
pads = _pypads_env.pypads
logger.info("Tracking execution to run with id " + pads.api.active_run().info.run_id)
dependencies = DependencyTO(parent=_logger_output)
try:
# Execute pip freeze
try:
# noinspection PyProtectedMember,PyPackageRequirements
from pip._internal.operations import freeze
except ImportError: # pip < 10.0
# noinspection PyUnresolvedReferences,PyPackageRequirements
from pip.operations import freeze
dependencies.add_dependency(list(freeze.freeze()))
except Exception as e:
_logger_output.set_failure_state(e)
finally:
_logger_output.dependencies = dependencies.store()
def is_dev_mode() -> bool:
"""Detects if flambe was installed in editable mode.
For more information:
https://pip.pypa.io/en/latest/reference/pip_install/#editable-installs
Returns
-------
bool
"""
x = freeze.freeze()
for pkg in x:
if pkg.startswith("-e") and pkg.endswith("egg=flambe"):
return True
return False
def pip_freeze():
result = getattr(pip_freeze, '_result', None)
if result is not None:
return result
# Read installed packages from pip freeze
from pip._internal.operations.freeze import freeze
distinfo = []
h = md5()
for line in freeze():
line = line.strip().lower()
if line == '':
continue
h.update(line.encode('utf-8'))
dinfo = None
mpkg = re.match(r'(.+)==(.+)', line)
if mpkg:
dinfo = DistInfo(name=mpkg.group(1),
version=mpkg.group(2),
commit=None)
mgit = re.match(r'-e\sgit\+.+\@(.{8}).{32}\#egg=(\w+).*$', line)
if mgit:
dinfo = DistInfo(name=mgit.group(2),
version=get_distribution(mgit.group(2)).version,
commit=mgit.group(1))
if dinfo is not None:
distinfo.append(dinfo)
else:
_logger.warn("Could not parse pip freeze line: %s", line)
static_key = h.hexdigest()[:8]
def _sort_key(di):
d = get_distribution(di.name)
ep = len(d.get_entry_map('nextgisweb.packages')) != 0
return (not ep, di.name)
distinfo = sorted(distinfo, key=_sort_key)
result = (static_key, tuple(distinfo))
setattr(pip_freeze, '_result', result)
return result
def _install_packages(self, dependent_pkg_path, pkg_to_exclude):
python_executable = self.context.venv.python_executable
working_dir = self.context.package_root_path
temp_dir = self.context.temp_path
list_to_exclude = [
pkg_to_exclude,
]
installed_pkgs = [
p.split('==')[0]
for p in list(freeze.freeze(paths=self.context.venv.lib_paths))
if p.startswith('azure-')
]
logging.info("Installed azure sdk packages:{}".format(installed_pkgs))
# Do not exclude list of packages in tools directory and so these tools packages will be reinstalled from repo branch we are testing
root_path = os.path.abspath(
os.path.join(dependent_pkg_path, "..", "..", ".."))
tools_packages = find_tools_packages(root_path)
installed_pkgs = [
req for req in installed_pkgs if req not in tools_packages
]
list_to_exclude.extend(installed_pkgs)
# install dev requirement but skip already installed package which is being tested or present in dev requirement
filtered_dev_req_path = filter_dev_requirements(
dependent_pkg_path, list_to_exclude, dependent_pkg_path)
if filtered_dev_req_path:
logging.info("Installing filtered dev requirements from {}".format(
filtered_dev_req_path))
run_check_call(
[
python_executable, "-m", "pip", "install", "-r",
filtered_dev_req_path
],
dependent_pkg_path,
)
else:
logging.info("dev requirements is not found to install")
# install dependent package which is being verified
run_check_call(
[python_executable, "-m", "pip", "install", dependent_pkg_path],
temp_dir)
def main():
log.debug('*** System Information ***')
log.debug('Platform: %s' % sys.platform)
log.debug('Python: %s' % str(sys.version).replace('\n', ''))
log.debug('Pip: %s' % ', '.join(freeze.freeze()))
log.info("Running lcmap-tap version %s" % __version__)
# Create a QApplication object, necessary to manage the GUI control flow and settings
app = QApplication(sys.argv)
# session_id = "session_{}".format(MainControls.get_time())
control_window = MainControls()
if control_window:
# Enter the main event loop, begin event handling for application widgets until exit() is called
sys.exit(app.exec_())
def validate_environment(output_dir: Path):
original_environment_file = output_dir / 'requirements.txt'
current_environment_list = [p for p in freeze.freeze()]
if not original_environment_file.exists(): # original run
with original_environment_file.open('w') as f:
f.write('\n'.join(current_environment_list))
else: # compare with original
with original_environment_file.open('r') as f:
original_environment_list = [line.replace('\n', '') for line in f]
current_environment = convert_pip_list_to_dict(
current_environment_list)
original_environment = convert_pip_list_to_dict(
original_environment_list)
compare_environments(current_environment, original_environment)
logger.info(
'Validation of environment successful. All pip installed packages match '
'original versions. Run can proceed.')
def run(self, options, args):
format_control = index.FormatControl(set(), set())
wheel_cache = WheelCache(options.cache_dir, format_control)
skip = set(stdlib_pkgs)
if not options.freeze_all:
skip.update(DEV_PKGS)
freeze_kwargs = dict(
requirement=options.requirements,
find_links=options.find_links,
local_only=options.local,
user_only=options.user,
skip_regex=options.skip_requirements_regex,
isolated=options.isolated_mode,
wheel_cache=wheel_cache,
skip=skip,
exclude_editable=options.exclude_editable)
for line in freeze(**freeze_kwargs):
sys.stdout.write(line + '\n')
def print_versions(output, libraries=[]):
try:
from pip._internal.operations import freeze
except ImportError:
from pip.operations import freeze
if libraries:
click.echo(
"Warning: The libraries argument has no effect",
err=True,
)
deps = freeze.freeze()
if output:
with open(output, 'w') as f:
for dep in deps:
print(dep, file=f)
else:
for dep in deps:
click.echo(dep)
