def add_requirements(self, raw, insert=False, commit=True):
old = Requirements(home=self.home)
path = os.path.join(self.path, 'requirements.txt')
if os.path.exists(path):
old.parse_file(path)
new = Requirements(home=self.home, file=StringIO(raw))
new_urls = set()
new_names = set()
for req in new.iter_packages():
new_names.add(req.name or guess_name(req.url))
new_urls.add(req.url)
for prefix, element, postfix in old:
if (not isinstance(element, Package) or
(element.name or guess_name(element.url)) not in new_names
or element.url not in new_urls):
if insert:
new.append((prefix, element, postfix))
else:
new.insert(0, (prefix, element, postfix))
with open(path, 'wb') as fh:
for line in new.iter_dump():
fh.write(line)
if commit:
self.commit('add requirements')
def add_requirements(self, raw, insert=False, commit=True):
old = Manifest(home=self.home)
path = os.path.join(self.path, 'manifest.txt')
if os.path.exists(path):
old.parse_file(path)
raw = raw.decode() if six.PY2 else raw
new = Manifest(home=self.home, file=StringIO(raw))
new_urls = set()
new_names = set()
for req in new.iter_packages():
new_names.add(req.name or guess_name(req.url))
new_urls.add(req.url)
for item in old._items:
element = item.value
if (not isinstance(element, Package) or
(element.name or guess_name(element.url)) not in new_names
or element.url not in new_urls):
if insert:
new._append(element)
else:
new._insert(0, element)
with open(path, 'w') as fh:
for line in new.iter_dump():
fh.write(line)
if commit:
self.commit('add requirements')
def iter_dump(self):
# We track the state of the environment as we progress, and don't
# include envvars in each requirement if they exactly match those
# in the current state.
environ = {}
for item in self._items:
# TODO: Refactor.
element = item.value
if isinstance(element, Envvar):
environ[element.name] = element.value
if isinstance(element, Package):
# Get a copy to mutate.
pkg = element = element.copy()
# We don't need a name if it matches the guessed version.
if pkg.name and pkg.name == guess_name(pkg.url):
pkg.name = None
# Strip out anything in the base environment which matches.
for k, v in environ.items():
if pkg.base_environ.get(k) == v:
del pkg.base_environ[k]
yield '%s%s%s\n' % (item.prefix or '', element or '', item.suffix
or '')
def resolve(self, req, check_existing=True, weak=False, env=None):
# We may need to guess a name.
name = req.name or guess_name(req.url)
try:
return self[name]
except KeyError:
pass
# We don't want to mutate the incoming package, even though we could,
# because we want to allow others to keep the abstract and concrete
# packages isolated if they want to.
pkg = req.copy(set=self)
# Our guessed name may be wrong after the package is resolved (e.g.
# deferred packages will definitely be wrong).
pkg.name = pkg.name or name
if check_existing:
(pkg.resolve_existing(env=env or self.env)
or pkg.resolve_existing()
or (weak and pkg.resolve_existing(weak=True)))
# Store it under the package name since deferred dependencies will not
# have a name set (in order to load the specific package they were before).
self[pkg.name] = pkg
return pkg
def iter_dump(self, freeze=False):
# We track the state of the environment as we progress, and don't
# include envvars in each requirement if they exactly match those
# in the current state.
environ = {}
for before, element, after in self:
if isinstance(element, Envvar):
environ[element.name] = element.value
if isinstance(element, Package):
req = element = (element.freeze()
if freeze else element.copy())
# We don't need a name if it matches the guessed version.
if req.name and req.name == guess_name(req.url):
req.name = None
# Strip out anything in the base environment which matches.
for k, v in environ.iteritems():
if req.base_environ.get(k) == v:
del req.base_environ[k]
yield '%s%s%s\n' % (before or '', element, after or '')
def _guess_names(self, strict=True):
"""Guess names for every requirement which does not already have one.
This mutates the requirements as it goes; if it fails then some
requirements will have already had their name set.
"""
names = set()
to_guess = []
# First pass: the explicitly named.
for req in self.iter_packages():
if not req.name:
to_guess.append(req)
continue
if req.name.lower() in names:
raise ValueError(
'name collision; please rename one of the %rs' % req.name)
names.add(req.name.lower())
# Second pass; the rest.
for req in to_guess:
name = guess_name(req.url)
if name.lower() in names:
if strict:
raise ValueError(
'name collision; please set name for one of the %rs' %
name)
else:
names.add(name.lower())
req.name = name
def test_guess_name(self):
self.assertEqual(
guess_name('https://github.com/exampleorg/example.git'),
'example',
)
self.assertEqual(
guess_name('[email protected]:exampleorg/example.git'),
'example',
)
self.assertEqual(
guess_name('[email protected]:exampleorg/prefix-example.git'),
'prefix-example',
)
self.assertEqual(
guess_name('[email protected]:exampleorg/example'),
'example',
)
self.assertEqual(
guess_name('https://pypi.python.org/packages/source/e/example/example-1.0.0.tar.gz#md5=something'),
'example',
)
self.assertEqual(
guess_name('http://example.org/download/example/Example-v1.0.0-rc1.tar.gz?key=value'),
'example',
)
self.assertEqual(
guess_name('homebrew+sqlite3'),
'sqlite3',
)
self.assertEqual(
guess_name('homebrew:sqlite3'),
'sqlite3',
)
self.assertEqual(
guess_name('git+https://github.com/PixarAnimationStudios/OpenSubdiv'),
'opensubdiv',
)
def __init__(self,
args=None,
*,
home=None,
set=None,
dev=False,
context=None,
parent=None,
source=None,
**kwargs):
super(Package, self).__init__()
source = source or parent
# Must be early due to some properties using this.
self.home = home = home or (source.home if source else None)
if not home:
raise ValueError("Package requires home")
self.context = context = context or (source.context
if source else None)
if not context and not dev:
raise ValueError(
"Package requires context (Manifest) when not dev")
if args and kwargs:
raise ValueError('specify either args OR kwargs')
if isinstance(args, self.__class__):
kwargs = args.to_kwargs()
args = None
elif isinstance(args, dict):
kwargs = args
args = None
if args:
if isinstance(args, six.string_types):
args = shlex.split(args)
if isinstance(args, (list, tuple)):
try:
requirement_parser.parse_args(args, namespace=self)
except RequirementParseError as e:
raise RequirementParseError("%s in %s" % (e.args[0], args))
elif isinstance(args, argparse.Namespace):
for action in requirement_parser._actions:
name = action.dest
setattr(self, name, getattr(args, name))
else:
raise TypeError(
"args must be one of (str, list, tuple, dict); got {}".
format(args.__class__))
else:
for action in requirement_parser._actions:
name = action.dest
# Version is a bit special, and should not have a default applied
# here, otherwise to_kwargs will clear it out.
if name in ('version', ):
try:
value = kwargs.pop(name)
except KeyError:
continue
else:
value = kwargs.pop(name, action.default)
setattr(self, name, value)
if kwargs:
raise ValueError("too many kwargs: {}".format(list(kwargs)))
assert self.url
# Assert we have a name.
if self.name:
if self.name.lower() != self.name:
log.warning("package name {!r} was not lowercase".format(
self.name))
self.name = self.name.lower()
else:
self.name = guess_name(self.url)
# TODO: Deprecate these.
self.dependencies = []
self.set = set
# Variant relationships.
self.parent = parent
self._children = None
self._child_is_self = None
# Make sure to make copies of anything that is mutable.
self.base_environ = self.base_environ.copy(
) if self.base_environ else {}
self.environ = self.environ.copy() if self.environ else {}
self.config = self.config[:] if self.config else []
# Initialize other state not covered by the argument parser.
# TODO: Should this come from the parent?
self.link_id = None
self.package_name = self.build_name = None
self.package_path = self.build_path = self.install_path = None
# Share some state with the parent.
if parent:
self.meta = parent.meta # Directly shared.
self.pipeline = parent.pipeline.copy(self)
else:
self.meta = context.load_meta(self.name) if context else None
self.url = self.get_meta('url') or self.url
self.version = self.get_meta('version') or self.version
self._init_pipeline(dev=dev)
def test_guess_name_regressions(self):
self.assertEqual(
guess_name('https://pypi.python.org/packages/2.7/S/Sphinx/Sphinx-1.3.1-py2.py3-none-any.whl'),
'sphinx',
)
def add(args):
home = args.assert_home()
env_repo = home.get_env_repo(args.repo)
req_set = env_repo.load_requirements()
pkg_set = PackageSet(home=home)
baked_any = None
if args.update:
baked_any = False
for req in req_set.iter_packages():
pkg = pkg_set.resolve(req, check_existing=False)
if pkg.fetch_type != 'git':
continue
print style_note('Fetching', str(req))
pkg.repo.fetch('origin',
'master') # TODO: track these another way?
if pkg.repo.check_ff_safety('origin/master'):
pkg.repo.checkout('origin/master')
head = pkg.repo.head[:8]
if head != req.revision:
req.revision = pkg.repo.head[:8]
print style_note('Updated', str(req))
baked_any = True
if args.bake_installed:
baked_any = False
for req in req_set.iter_packages():
pkg = pkg_set.resolve(req)
if pkg.fetch_type != 'git':
continue
repo = pkg.pipeline.steps['fetch'].repo
if req.name and req.name == guess_name(req.url):
req.name = None
baked_any = True
print style_note('Unset redundant name', req.name)
if pkg.installed and req.revision != repo.head[:8]:
req.revision = repo.head[:8]
baked_any = True
print style_note('Pinned', req.name, req.revision)
if args.checksum:
baked_any = False
for req in req_set.iter_packages():
pkg = pkg_set.resolve(req)
if pkg.checksum:
continue
if not pkg.package_path or not os.path.isfile(pkg.package_path):
continue
req.checksum = checksum_file(pkg.package_path)
print style_note('Checksummed', pkg.name, req.checksum)
baked_any = True
if baked_any is not None:
if baked_any:
env_repo.dump_requirements(req_set)
else:
print style_note('No changes.')
return
row = home.get_development_record(os.path.abspath(args.package))
if not row:
raise ValueError('No development package %r' % args.package)
dev_repo = GitRepo(row['path'])
# Get the normalized origin.
dev_remote_urls = set()
for url in dev_repo.remotes().itervalues():
url = normalize_git_url(url, prefer='scp') or url
log.debug('adding dev remote url: %s' % url)
dev_remote_urls.add(url)
if not dev_remote_urls:
print style_error('No git remotes for %s' % row['path'])
return 1
for req in req_set.iter_packages(eval_control=False):
# We only deal with git packages.
pkg = pkg_set.resolve(req, check_existing=False)
if pkg.fetch_type != 'git':
continue
req_url = normalize_git_url(req.url, prefer='scp')
log.debug('does match package url?: %s' % req_url)
if req_url in dev_remote_urls:
if req.revision == dev_repo.head[:8]:
print style_note('No change to', str(req))
else:
req.revision = dev_repo.head[:8]
print style_note('Updated', str(req))
break
else:
if not args.init:
print '{error}: No required package {name}; would match one of:'.format(
error=style('Error', 'red'),
name=style(args.package, bold=True))
for url in sorted(dev_remote_urls):
print ' {}'.format(url)
print 'Use {} to setup: git+{} --revision {}'.format(
style('vee add --init %s' % args.package, 'green'),
dev_repo.remotes()['origin'], dev_repo.head[:8])
return 1
req = Package(
url=normalize_git_url(dev_repo.remotes()['origin'], prefix=True),
revision=dev_repo.head[:8],
home=home,
)
req_set.append(('', req, ''))
env_repo.dump_requirements(req_set)