def relocate(root, con, spec=None, dry_run=False, target_cache=None): target_cache = {} if target_cache is None else target_cache flags, include, exclude = _parse_spec(spec, root) if 'linux' in flags and not sys.platform.startswith('linux'): return if 'darwin' in flags and sys.platform != 'darwin': return if not (include or 'auto' in flags): raise ValueError('no libraries to include') # Find everything in include on OS X, since we need to actually find the # individual dependencies. if sys.platform == 'darwin': for path in include: for found in find_shared_libraries(path): log.debug('found %s' % found) for name in name_variants(os.path.basename(found)): target_cache.setdefault(name, []).append(found) for lib_path in find_shared_libraries(root): log.info(lib_path) with log.indent(): if sys.platform == 'darwin': _relocate_darwin_library(lib_path, con, flags, include, exclude, dry_run, target_cache) else: _relocate_linux_library(lib_path, include, dry_run) if 'pkgconfig' in flags: relocate_pkgconfig(root)
def iter_development_packages(self, exists=True, search=True): if not search: # We used to have a development_packages table, and searching was # something that had to be requested. log.debug( "iter_development_packages(..., seach=False) is deprecated.") for root in self.dev_search_path: if not os.path.exists(root): continue for name in os.listdir(root): path = os.path.join(root, name) if name.endswith('.vee-dev.json'): yield DevPackage.from_tag(path, home=self) continue # Not used yet. sub_path = os.path.join(path, '.vee-dev.json') if os.path.exists(sub_path): yield DevPackage.from_tag(sub_path, home=self)
def get_installed_shared_libraries(con, package_id, install_path, rescan=False): with con: # Determine if we should scan. do_scan = rescan if not rescan: row = con.execute('SELECT scanned_for_libraries FROM packages WHERE id = ?', [package_id]).fetchone() do_scan = not row[0] # Return existing results if we should not scan. if not do_scan: cur = con.execute('SELECT path FROM shared_libraries WHERE package_id = ?', [package_id]) return [row[0] for row in cur] # Blow out any existing results, and then scan. cur = con.execute('DELETE FROM shared_libraries WHERE package_id = ?', [package_id]) res = [] for lib_path in find_shared_libraries(install_path): log.debug('Found shared library %s' % lib_path) res.append(lib_path) con.execute('''INSERT INTO shared_libraries (package_id, name, path) VALUES (?, ?, ?)''', [ package_id, os.path.basename(lib_path), lib_path, ]) con.execute('UPDATE packages SET scanned_for_libraries = 1 WHERE id = ?', [package_id]) return res
def persist_in_db(self, con=None, force=False): if not self.is_dirty and not force: return self.id data = {} for col in self.__columns__: try: if col._persist: data[col.name] = col._persist(self) elif col._getter: data[col.name] = col._getter(self) else: data[col.name] = self.__dict__[col.name] except KeyError: pass con = con or self._connect() if self.id: con.update(self.__tablename__, data, {'id': self.id}) else: self.id = con.insert(self.__tablename__, data) log.debug('%s added to %s with ID %d' % (self.__class__.__name__, self.__tablename__, self.id)) self.is_dirty = False return self.id
def inspect(self): pkg = self.package if self.setup_path and not self.egg_path: log.info(style_note('Building Python egg-info')) res = call_setup_py(self.setup_path, ['egg_info'], env=pkg.fresh_environ(), indent=True, verbosity=1) if res: raise RuntimeError('Could not build Python package') self.egg_path = find_in_tree(pkg.build_path, '*.egg-info', 'dir') if not self.egg_path: log.warning('Could not find newly created *.egg-info') if self.egg_path: requires_path = os.path.join(self.egg_path, 'requires.txt') if os.path.exists(requires_path): for line in open(requires_path, 'rb'): line = line.strip() if not line: continue if line.startswith('['): break name = re.split('\W', line)[0].lower() log.debug('%s depends on %s' % (pkg.name, name)) pkg.dependencies.append( Package(name=name, url='pypi:%s' % name))
def get_env_repo(self, name=None): name = name or self.default_repo_name if name: row = self.db.execute( 'SELECT * FROM repositories WHERE name = ? LIMIT 1', [name]).fetchone() if not row: raise ValueError('%r repo does not exist' % name) else: # Grab the default repo if possible, otherwise make sure there is # only one. rows = self.db.execute( 'SELECT * FROM repositories ORDER BY is_default DESC LIMIT 2' ).fetchall() if not rows: raise ValueError('no repositories exist') elif rows[0]['is_default']: row = rows[0] elif len(rows) == 1: row = rows[0] else: raise ValueError('multiple repositories with no default') env_repo = EnvironmentRepo(row, home=self) if not env_repo.exists: log.debug('Looking for env_repo: %s' % env_repo.work_tree) raise ValueError('%r repo does not exist' % env_repo.name) return env_repo
def edit(args): home = args.assert_home() repo = home.get_repo(args.repo) cmd = [] cmd.extend(shlex.split(os.environ.get('EDITOR', 'vim'))) cmd.append(os.path.join(repo.work_tree, 'manifest.txt')) log.debug(cmd, verbosity=1) os.execvp(cmd[0], cmd)
def edit(args): home = args.assert_home() env_repo = home.get_env_repo(args.repo) cmd = [] cmd.extend(shlex.split(os.environ['EDITOR'])) cmd.append(os.path.join(env_repo.work_tree, 'requirements.txt')) log.debug(cmd, verbosity=1) os.execvp(cmd[0], cmd)
def insert(self, table, data, on_conflict=None): pairs = sorted(data.iteritems()) query = 'INSERT %s INTO %s (%s) VALUES (%s)' % ( 'OR ' + on_conflict if on_conflict else '', escape_identifier(table), ','.join(escape_identifier(k) for k, v in pairs), ','.join('?' for _ in pairs), ) params = [v for k, v in pairs] log.debug('%s %r' % (query, params)) self.execute(query, params) return self.lastrowid
def vee(args, environ=None, check=True, stdout=False): full_environ = os.environ.copy() full_environ.update(environ or {}) full_environ.update(_environ_diff) log.debug('$ vee ' + ' '.join(args), name='vee.tests') if stdout: cmd = ['vee'] cmd.extend(args) return subprocess.check_output(cmd, env=environ) res = _main(args, environ=full_environ) if check and res: raise ValueError('return code %d' % res) return res
def call(cmd, **kwargs): # Log the call. kwargs.pop('silent', None) # B/C. VEE = os.environ.get('VEE') cmd_collapsed = [x.replace(VEE, '$VEE') if VEE else x for x in cmd] log.debug( '$ ' + ' '.join(cmd_collapsed), verbosity=2, _frame=kwargs.pop('_frame', 0) + 3 ) check = kwargs.pop('check', True) verbosity = kwargs.pop('verbosity', 0) indent = kwargs.pop('indent', False) if indent: indent = log.indent() indent.__enter__() pty = kwargs.pop('pty', None) stdout = _CallOutput(kwargs.pop('stdout', None), 'stdout', verbosity, pty=pty) stderr = _CallOutput(kwargs.pop('stderr', None), 'stderr', verbosity, pty=pty) proc = subprocess.Popen(cmd, stdout=stdout.slave_fd, stderr=stderr.slave_fd, bufsize=0, **kwargs) stdout.start(proc) stderr.start(proc) proc.wait() stdout.join() stderr.join() if indent: indent.__exit__(None, None, None) if (check or stdout.return_buffer or stderr.return_buffer) and proc.returncode: raise subprocess.CalledProcessError(proc.returncode, cmd) if stdout.return_buffer and stderr.return_buffer: return ''.join(stdout.buffer), ''.join(stderr.buffer) if stdout.return_buffer: return ''.join(stdout.buffer) if stderr.return_buffer: return ''.join(stderr.buffer) return proc.returncode
def rewrite_shebang(self, old_path, new_path): # Only care if it is at all executable. stat = os.stat(old_path) if not (stat.st_mode & 0o111): return # If it starts with a Python shebang, rewrite it. with open(old_path, 'rb') as old_fh: old_shebang = old_fh.readline() m = re.match(r'#!(|\S+/)([^\s/]+)', old_shebang) if not m: return new_bin = os.path.join(self.path, 'bin', m.group(2)) if not os.path.exists(new_bin): return new_shebang = '#!%s%s' % (new_bin, old_shebang[m.end(2):]) log.info('Rewriting shebang of %s' % old_path, verbosity=1) log.debug('New shebang: %s' % new_shebang.strip(), verbosity=1) self._assert_real_dir(os.path.dirname(new_path)) # Due to the way the _assert_real_dir works, we may have already # created a symlink in the location of the new_path which points to # the old_path. If we don't delete it first, then we will be # reading and writing to the same time, and will only get the # shebang + 1024 bytes (the buffer size on my machine). if os.path.lexists(new_path): os.unlink(new_path) with open(new_path, 'wb') as new_fh: new_fh.write(new_shebang) new_fh.writelines(old_fh) try: shutil.copystat(old_path, new_path) except OSError as e: # These often come up when you are not the owner # of the file. log.exception('Could not copystat to %s' % new_path) if e.errno != errno.EPERM: raise return True
def environ_diff(self): if self._environ_diff is None: self._environ_diff = {} for e in (self.base_environ, self.environ): for k, v in e.items(): self._environ_diff[k] = self.render_template(v, name=k) # Just for debugging... for k, v in sorted(self._environ_diff.items()): old_v = os.environ.get(k) if old_v is not None: v = v.replace(old_v, '@') v = v.replace(self.home.root, '$VEE') log.debug('%s %s=%s' % (style('setenv', 'blue'), k, v), verbosity=2) return self._environ_diff
def fetch(self): pkg = self.package meta = self._meta() all_releases = [(Version(v), rs) for v, rs in meta['releases'].iteritems()] all_releases.sort(reverse=True) if pkg.revision: expr = VersionExpr(pkg.revision) matching_releases = [(v, rs) for v, rs in all_releases if expr.eval(v)] log.debug( '%s matched %s' % (expr, ','.join(str(v) for v, _ in matching_releases) or 'none')) else: matching_releases = all_releases for version, releases in matching_releases: release = next( (r for r in releases if r['packagetype'] == 'sdist'), None) if release: break else: raise ValueError('no sdist %s %s on the PyPI;' % (self.name, expr if pkg.revision else '(any)')) pkg.revision = str(version) if release.get('md5_digest'): pkg.checksum = 'md5:%s' % release['md5_digest'] pkg.package_name = os.path.join(self.name, os.path.basename(release['url'])) pkg._assert_paths(package=True) if os.path.exists(pkg.package_path): log.info(style_note('Already downloaded', release['url'])) return log.info(style_note('Downloading', release['url'])) download(release['url'], pkg.package_path)
def persist_in_db(self, con=None): self._set_names(package=True, build=True, install=True) if not self.installed: log.warning('%s does not appear to be installed to %s' % (self.name, self.install_path)) raise ValueError('cannot record requirement that is not installed') con = con or self.home.db.connect() with con: exists = self.id is not None res = super(Package, self).persist_in_db(con=con) if exists: con.execute('DELETE FROM package_dependencies WHERE depender_id = ?', [self.id]) for dep in self.dependencies: dep_id = dep.id_or_persist(con=con) log.debug('Recorded %s -> %s dependency as %d' % ( self.name, dep.name, dep_id )) con.execute('INSERT INTO package_dependencies (depender_id, dependee_id) VALUES (?, ?)', [ self.id, dep_id ]) return res
def load(self, step_name): try: return self.steps[step_name] except KeyError: pass # See if any previous steps provide it. step_i = self._step_index[step_name] for i in xrange(step_i - 1, -1, -1): prev_name = self._step_names[i] prev_step = self.steps[prev_name] step = prev_step.get_next(step_name) if step: log.debug('%s (%s) provided sucessor %s (%s) for %s' % ( prev_step.name, prev_name, step.name, step_name, self._package ), verbosity=2) self.steps[step_name] = step return step # Load the step classes. if not _step_classes: for ep in pkg_resources.iter_entry_points('vee_pipeline_steps'): cls = ep.load() cls.name = ep.name _step_classes.append(cls) _step_classes_by_name[ep.name] = cls _step_classes.sort(key=lambda cls: getattr(cls, 'factory_priority', 1), reverse=True) # Find something that self-identifies it provides this step. for cls in _step_classes: step = cls.factory(step_name, self._package) if step: log.debug('%s factory built %s for %s' % (step.name, step_name, self._package), verbosity=2) self.steps[step_name] = step return step raise ValueError('Cannot load %s step for %s' % (step_name, self._package.freeze()))
def resolve_existing(self, env=None, weak=False): """Check against the database to see if this was already installed.""" if self.id is not None: raise ValueError('requirement already in database') cur = self.home.db.cursor() # Dependencies are deferred. deferred = self.url.startswith('deferred:') if deferred: deferred_id = int(self.url.split(':')[1]) cur.execute('SELECT * from packages WHERE id = ?', [deferred_id]) else: clauses = ['install_path IS NOT NULL'] values = [] if not weak and self.url: clauses.append('url = ?') values.append(self.url) for name in ('name', 'etag', 'install_name'): if getattr(self, name): clauses.append('%s = ?' % name) values.append(getattr(self, name)) clause = ' AND '.join(clauses) # log.debug('SELECT FROM packages WHERE %s' % ' AND '.join('%s = %r' % (c.replace(' = ?', ''), v) for c, v in zip(clauses[1:], values)), verbosity=2) if env: values.append(env.id_or_persist()) cur.execute( ''' SELECT packages.*, links.id as link_id FROM packages LEFT OUTER JOIN links ON packages.id = links.package_id WHERE %s AND links.environment_id = ? ORDER BY links.created_at DESC, packages.created_at DESC ''' % clause, values) else: cur.execute( ''' SELECT * FROM packages WHERE %s ORDER BY packages.created_at DESC ''' % clause, values) for row in cur: # Make sure it has enough provisions. provides = Provision(row['provides']) if any( provides.get(key, None) != value for key, value in self.provides.items()): log.debug( 'Found %s (%d) whose provisions %s do not satisfy %s' % ( self.name or row['name'], row['id'], row['provides'], self.provides, ), verbosity=2) continue # Make sure it has enough requirements. requires = RequirementSet(row['requires']) reqs_ok = True for name in self.requires: try: reqs = requires[name] except KeyError: reqs_ok = False continue if any( reqs.get(key, None) != value for key, value in self.requires.items()): reqs_ok = False continue if not reqs_ok: log.debug( 'Found %s (%d) whose requirements %s do not satisfy %s' % ( self.name or row['name'], row['id'], row['requires'], self.requires, ), verbosity=2) if not os.path.exists(row['install_path']): log.warning( 'Found %s (%d) does not exist at %s' % (self.name or row['name'], row['id'], row['install_path'])) continue break else: if deferred: raise ValueError( 'deferred package %d no longer exists; consider `vee gc`' % deferred_id) return log.debug('Found %s (%d%s%s) at %s' % ( self.name or row['name'], row['id'], ' weakly' if weak else '', ' in env %d' % env.id if env else '', row['install_path'], )) self.restore_from_row(row) self.link_id = row.get('link_id') if deferred: self._init_pipeline() self._load_dependencies(cur) return True
def _install_one(self, names, name, link_env, reinstall, relink, no_deps): try: pkg = self[name] except KeyError: # print(', '.join(sorted(self.keys()))) raise reinstall_this = name in reinstall relink_this = name in relink if name not in self._extracted: try: # Between every step, take a look to see if we now have # enough information to tell that it is already installed. pkg.assert_uninstalled(uninstall=reinstall_this) pkg.pipeline.run_to('fetch') pkg.assert_uninstalled(uninstall=reinstall_this) pkg.pipeline.run_to('extract') pkg.assert_uninstalled(uninstall=reinstall_this) pkg.pipeline.run_to('inspect') pkg.assert_uninstalled(uninstall=reinstall_this) except AlreadyInstalled: self._installed.add(name) finally: self._extracted.add(name) # Loop around for dependencies. We insert dependencies, and the # package itself, back into the names to check. If we get back to # a name that we have already deferred in this manner, we continue # anyways, since that means there is a dependency cycle. We assume # that dependency order is resolved in the requirements file. deferred = False deps_installed = True insert_i = 0 for i, dep in ([] if no_deps else enumerate(pkg.dependencies)): # Since resolution is rather loose in here (only by name, not URL) # we want to replace dependencies with their concrete variant to # ease recording that into the database. dep = self.resolve(dep, weak=True) pkg.dependencies[i] = dep self._parent_names.setdefault(dep.name, pkg.name) # We must check if the dependency errored, otherwise we will still # end up in an infinite loop. if dep.name in self._errored: log.warning('Skipping due to error in %s' % dep.name) self._errored.add(pkg.name) return if dep.name not in self._installed: key = (name, dep.name) if key not in self._deferred: log.debug('%s needs %s; deferring install' % (name, dep.name)) self._deferred.add(key) deferred = True else: log.debug('%s needs %s, but install was already deferred' % (name, dep.name)) deps_installed = False names.insert(insert_i, dep.name) insert_i += 1 if deferred: names.insert(insert_i, name) return pre_build_deps = pkg.dependencies[:] if name not in self._installed: try: pkg.pipeline.run_to('build') pkg.pipeline.run_to('install') pkg.pipeline.run_to('relocate') except AlreadyInstalled: pass pkg.pipeline.run_to('optlink') self._installed.add(name) # We need to build/install Homebrew packages before we can decide # which of their optional dependencies will be used. The relocation # process can also determine other dependencies. We need to run # these new ones through the pipe too. if pkg.dependencies != pre_build_deps: log.debug('%s has changed dependencies after build/install') names.insert(insert_i, name) return if pkg.virtual: return if name not in self._persisted: # We need to wait to persist until all dependencies are # installed. if not deps_installed: log.debug('%s cannot persist without all dependencies' % (name, )) names.insert(insert_i, name) return pkg.persist_in_db() pkg.shared_libraries() # TODO: Move this earlier? self._persisted.add(name) if link_env and name not in self._linked: try: pkg.link(link_env, force=relink_this) except AlreadyLinked: pass self._linked.add(name)
def call(cmd, **kwargs): # Log the call. kwargs.pop('silent', None) # B/C. VEE = os.environ.get('VEE') cmd_collapsed = [x.replace(VEE, '$VEE') if VEE else x for x in cmd] log.debug( '$ ' + ' '.join(cmd_collapsed), verbosity=2, _frame=kwargs.pop('_frame', 0) + 3 ) check = kwargs.pop('check', True) decode = kwargs.pop('decode', False) verbosity = kwargs.pop('verbosity', 0) indent = kwargs.pop('indent', False) if indent: indent = log.indent() indent.__enter__() pty = kwargs.pop('pty', None) stdout = _CallOutput(kwargs.pop('stdout', None), 'stdout', verbosity, pty=pty) stderr = _CallOutput(kwargs.pop('stderr', None), 'stderr', verbosity, pty=pty) if kwargs.pop('vee_in_env', False): vee_src = os.path.abspath(os.path.join(__file__, '..', '..')) env = kwargs.get('env', os.environ).copy() env['PYTHONPATH'] = join_env_path( vee_src, env.get('PYTHONPATH') ) env['PATH'] = join_env_path(os.path.join(vee_src, 'bin'), env.get('PATH')) kwargs['env'] = env proc = subprocess.Popen(cmd, stdout=stdout.slave_fd, stderr=stderr.slave_fd, bufsize=0, **kwargs) stdout.start(proc) stderr.start(proc) proc.wait() stdout.join() stderr.join() if indent: indent.__exit__(None, None, None) if (check or stdout.return_buffer or stderr.return_buffer) and proc.returncode: raise subprocess.CalledProcessError(proc.returncode, cmd) # Decode if requested. if stdout.return_buffer: stdout_res = b''.join(stdout.buffer) if decode: stdout_res = stdout_res.decode() if stderr.return_buffer: stderr_res = b''.join(stderr.buffer) if decode: stderr_res = stderr_res.decode() if stdout.return_buffer and stderr.return_buffer: return stdout_res, stderr_res if stdout.return_buffer: return stdout_res if stderr.return_buffer: return stderr_res return proc.returncode
def inspect(self, pkg): if self.setup_path: stdout = call_setup_py(self.setup_path, ['egg_info'], env=pkg.fresh_environ(), stdout=True).decode() m = re.search(r'writing requirements to (.+?)\n', stdout) if not m: log.debug("No requirements") return requirements_path = os.path.join(os.path.dirname(self.setup_path), m.group(1)) for line in open(requirements_path): # Stop once we get to the "extras". if line.startswith('['): break m = re.match(r'^([\w\.-]+)', line) if m: name = m.group(1).lower() log.debug('%s depends on %s' % (pkg.name, name)) pkg.add_dependency(name=name, url='pypi:%s' % name.lower()) if self.dist_info_dir: for line in open(os.path.join(self.dist_info_dir, 'METADATA')): line = line.strip() if not line: break # We're at the end of the headers. key, value = line.split(': ', 1) key = key.lower() if key == 'requires-dist': # Environmental markers look like `FOO; extra == 'BAR'`. if ';' in value: value, raw_marker = value.split(';') value = value.strip() # We delay the import just in case the bootstrap is borked. from packaging.markers import Marker marker = Marker(raw_marker) if not marker.evaluate({'extra': None}): continue m = re.match(r'([\w-]+)(?:\s+\(([^)]+)\))?', value) if not m: log.warning( 'Could not parse requires-dist {!r}'.format(value)) continue dep_name, version_expr = m.groups() pkg.add_dependency( name=dep_name, url='pypi:{}'.format(dep_name), version=version_expr, )
def _relocate_darwin_library(lib_path, con, flags, include, exclude, dry_run, target_cache): auto = 'auto' in flags lib_id, lib_deps = get_dependencies(lib_path) id_versions = set( name_variants(os.path.basename(lib_id), version_only=True)) if lib_id else set() lib_versions = set( name_variants(os.path.basename(lib_path), version_only=True)) cmd = ['install_name_tool'] if lib_id != lib_path: log.info('id %s' % (lib_path), verbosity=1) cmd.extend(('-id', lib_path)) lib_def, lib_undef = get_symbols(lib_path) for dep_i, dep_path in enumerate(lib_deps): if dep_path == lib_id: log.warning('The ID is included?! %s' % lib_path) cmd.extend(('-change', dep_path, lib_path)) continue # If the dependency is similarly named to the library itself, then we # assume it is its own dependency. Which I don't understand... dep_versions = set( name_variants(os.path.basename(dep_path), version_only=True)) if dep_versions.intersection(id_versions) or dep_versions.intersection( lib_versions): log.warning('Library depends on itself?! %s' % dep_path) cmd.extend(('-change', dep_path, lib_path)) continue do_exclude = any(dep_path.startswith(x) for x in exclude) if not do_exclude and os.path.exists(dep_path): log.debug('skipping %s' % dep_path) continue dep_name = os.path.basename(dep_path) targets = [] for variant in name_variants(dep_name): if variant in target_cache: targets.extend(target_cache[variant]) if auto: cur = con.execute( 'SELECT path FROM shared_libraries WHERE name = ? ORDER BY created_at DESC', [variant]) new_targets = target_cache.setdefault(variant, []) new_targets.extend([row[0] for row in cur]) targets.extend(new_targets) # Go searching for the "best" relocation target. # The one with the most defined symbols missing from the lib wins # (essentially; it is more complex then that below). We also, dubiously, # accept libraries which provide no matching symbols as long as they # don't introduct any conflicts. There are a TON of these in FFmpeg. best_score = -1 best_target = None seen_targets = set() for target in targets: if target in seen_targets: continue seen_targets.add(target) if not os.path.exists(target): continue tar_def, tar_undef = get_symbols(target) pros = len(tar_def.intersection(lib_undef)) shared = len(tar_def.intersection(lib_def)) cons = len(lib_undef.intersection(lib_def)) log.debug('+%d ~%d -%d %s' % (pros, shared, cons, target), verbosity=2) if pros - shared - cons > best_score: best_score = pros - shared - cons best_target = (pros, shared, cons, target) if best_target is None: log.warning('No relocation targets for %s' % dep_path) continue if best_score < 0: log.warning('No positive relocation targets for %s' % dep_path) continue if best_target[1] or best_target[2]: log.warning('Best target has %s collisions for %s' % (best_target[1] + best_target[2], dep_path)) target = best_target[3] log.info('change %s -> %s' % (dep_name, target), verbosity=1) cmd.extend(('-change', dep_path, target)) if len(cmd) > 1 and not dry_run: cmd.append(lib_path) s = os.stat(lib_path) if not s.st_mode & stat.S_IWUSR: os.chmod(lib_path, s.st_mode | stat.S_IWUSR) call(cmd) os.chmod(lib_path, s.st_mode) else: call(cmd)
def add(args): home = args.assert_home() env_repo = home.get_env_repo(args.repo) req_set = env_repo.load_requirements() pkg_set = PackageSet(home=home) baked_any = None if args.update: baked_any = False for req in req_set.iter_packages(): pkg = pkg_set.resolve(req, check_existing=False) if pkg.fetch_type != 'git': continue print style_note('Fetching', str(req)) pkg.repo.fetch('origin', 'master') # TODO: track these another way? if pkg.repo.check_ff_safety('origin/master'): pkg.repo.checkout('origin/master') head = pkg.repo.head[:8] if head != req.revision: req.revision = pkg.repo.head[:8] print style_note('Updated', str(req)) baked_any = True if args.bake_installed: baked_any = False for req in req_set.iter_packages(): pkg = pkg_set.resolve(req) if pkg.fetch_type != 'git': continue repo = pkg.pipeline.steps['fetch'].repo if req.name and req.name == guess_name(req.url): req.name = None baked_any = True print style_note('Unset redundant name', req.name) if pkg.installed and req.revision != repo.head[:8]: req.revision = repo.head[:8] baked_any = True print style_note('Pinned', req.name, req.revision) if args.checksum: baked_any = False for req in req_set.iter_packages(): pkg = pkg_set.resolve(req) if pkg.checksum: continue if not pkg.package_path or not os.path.isfile(pkg.package_path): continue req.checksum = checksum_file(pkg.package_path) print style_note('Checksummed', pkg.name, req.checksum) baked_any = True if baked_any is not None: if baked_any: env_repo.dump_requirements(req_set) else: print style_note('No changes.') return row = home.get_development_record(os.path.abspath(args.package)) if not row: raise ValueError('No development package %r' % args.package) dev_repo = GitRepo(row['path']) # Get the normalized origin. dev_remote_urls = set() for url in dev_repo.remotes().itervalues(): url = normalize_git_url(url, prefer='scp') or url log.debug('adding dev remote url: %s' % url) dev_remote_urls.add(url) if not dev_remote_urls: print style_error('No git remotes for %s' % row['path']) return 1 for req in req_set.iter_packages(eval_control=False): # We only deal with git packages. pkg = pkg_set.resolve(req, check_existing=False) if pkg.fetch_type != 'git': continue req_url = normalize_git_url(req.url, prefer='scp') log.debug('does match package url?: %s' % req_url) if req_url in dev_remote_urls: if req.revision == dev_repo.head[:8]: print style_note('No change to', str(req)) else: req.revision = dev_repo.head[:8] print style_note('Updated', str(req)) break else: if not args.init: print '{error}: No required package {name}; would match one of:'.format( error=style('Error', 'red'), name=style(args.package, bold=True)) for url in sorted(dev_remote_urls): print ' {}'.format(url) print 'Use {} to setup: git+{} --revision {}'.format( style('vee add --init %s' % args.package, 'green'), dev_repo.remotes()['origin'], dev_repo.head[:8]) return 1 req = Package( url=normalize_git_url(dev_repo.remotes()['origin'], prefix=True), revision=dev_repo.head[:8], home=home, ) req_set.append(('', req, '')) env_repo.dump_requirements(req_set)
def gc(args): home = args.assert_home() con = home.db.connect() with con: repo_ids = {} for row in con.execute('SELECT id, name from repositories'): repo_ids[row['name']] = row['id'] envs_by_id = {} log.info(style_note('Cleaning environments')) for row in con.execute( 'SELECT id, name, path, repository_id from environments ORDER BY created_at ASC' ): id_, name, path, repo_id = row if not os.path.exists(path): log.info('environment does not exist at %s; deleting' % (path)) if not args.dry_run: delete_environment(con, id_) continue # Track for later. envs_by_id.setdefault(repo_id, []).append((id_, name, path)) # The rest is making sure the repo_id and commit are correct. if repo_id: continue m = re.match(r'(\w+)/commits/([0-9a-f]{7,8}(?:-dirty)?)$', name) if not m: log.warning( '%s (%d) does not appear to be managed by git; skipping' % (name, id_)) continue repo_name, commit_name = m.groups() repo_id = repo_ids.get(repo_name) if not repo_id: log.warning('repo %s does not exist for %s (%d); skipping' % (repo_name, name, id_)) continue log.info('Fixing repo relationship for %s (%d)' % (name, id_)) if not args.dry_run: con.execute( 'UPDATE environments SET repository_id = ?, repository_commit = ? WHERE id = ?', [repo_id, commit_name, id_]) if args.prune_environments: log.info(style_note('Pruning old environments')) for repo_id, envs in sorted(envs_by_id.iteritems()): for id_, name, path in envs[:-args.keep_latest]: log.info('Deleting %s (%d)' % (name, id_)) if not args.dry_run: shutil.rmtree(path) delete_environment(con, id_) log.info(style_note('Cleaning installed packages')) package_ids = [] install_paths_to_id = {} for row in con.execute( 'SELECT id, name, install_path, build_path from packages ORDER by created_at DESC' ): id_, name, install_path, build_path = row log.debug('%s %s %s' % (id_, name, install_path)) if not os.path.exists(install_path): log.info('%s no longer exists at %s; deleting' % (name, install_path)) if not args.dry_run: delete_package(con, id_) continue real_id = install_paths_to_id.get(install_path) if real_id: log.info('%s %d is a duplicate of %s; deleting' % (name, id_, real_id)) # TODO: update any links or package_dependencies which to point to this. if not args.dry_run: delete_package(con, id_) continue install_paths_to_id[install_path] = id_ if args.prune_orphaned_packages: row = con.execute( 'SELECT count(1) FROM links WHERE package_id = ?', [id_]).fetchone() if not row[0]: log.info('%s (%d) is not linked; deleting' % (name, id_)) if not args.dry_run: if build_path and os.path.exists(build_path): shutil.rmtree(build_path) shutil.rmtree(install_path) delete_package(con, id_)