def get_fetcher_from_repository(package_name, package_root, python_settings):
package_json = os.path.join(package_root, PACKAGE_JSON)
with open(package_json, 'r') as fin:
js = json.loads(fin.read())
short_name = js.get('name').split('/')[-1]
if 'repository' not in js:
raise BuckitException(
"The repository section is missing in {}", package_json)
repository = js.get('repository')
repo_type = repository.get("type", "")
if 'pip_info' in repository:
pip_info = repository.get('pip_info')
dest_dir = os.path.join(
package_root, pip_info.get("install_subdir", short_name)
)
return (
PipFetcher(
package_name,
package_json,
pip_info.get("pip2_package", ""),
pip_info.get("pip2_version", ""),
pip_info.get("pip3_package", ""),
pip_info.get("pip3_version", ""),
pip_info.get("main_rule", ""),
pip_info.get("buck_deps", []),
python_settings,
), dest_dir
)
elif repo_type == "git":
dest_dir = os.path.join(package_root, short_name)
return (
GitFetcher(
package_name,
package_json,
repository.get("url", ""),
repository.get("commit", None),
repository.get("tag", None),
), dest_dir
)
elif repo_type == "tarball":
dest_dir = os.path.join(package_root, short_name)
return (
HttpTarballFetcher(
package_name,
package_json,
repository.get("url", ""),
repository.get("sha256", ""),
), dest_dir
)
else:
raise BuckitException(
"repository.type in {} must be either 'git', 'tarball', or 'pip'",
package_json)
def __init__(
self, package_name, json_file, pip2_package, pip2_version, pip3_package,
pip3_version, main_rule, buck_deps, python_settings
):
self.package_name = package_name
self.main_rule = main_rule
self.buck_deps = buck_deps
self.python2 = None
self.python3 = None
if pip2_package and python_settings.use_python2:
self.python2 = PipPythonSettings(
python_settings.python2_virtualenv_command,
python_settings.python2_virtualenv_root,
pip2_package,
pip2_version,
'py2',
)
if pip3_package and python_settings.use_python3:
self.python3 = PipPythonSettings(
python_settings.python3_virtualenv_command,
python_settings.python3_virtualenv_root,
pip3_package,
pip3_version,
'py3',
)
self.python2_files = {"srcs": {}, "bins": {}}
self.python3_files = {"srcs": {}, "bins": {}}
if not main_rule:
raise BuckitException(
'A main_rule attribute must be set in {}', json_file)
def parse_package_info(package_path):
"""
Try to get the package info from the package.json inside of package_path
Arguments:
package_path - The path to the package root that contains package.json
Returns:
PackageInfo object with properties from the package.json
"""
package_path = os.path.abspath(package_path)
json_path = os.path.join(package_path, PACKAGE_JSON)
try:
with open(json_path, 'r') as fin:
js = json.loads(fin.read())
buckit = js.get('buckit', {})
cell_name = buckit.get('cell_name', js['name'])
includes = buckit.get('includes', None)
if includes:
includes_info = IncludesInfo(
includes.get('path', None),
includes.get('whitelist_functions', [])
)
if not isinstance(includes_info.path, str):
raise BuckitException(
"buckit.includes in {} should be a string", json_path)
if not isinstance(includes_info.whitelist_functions, list):
raise BuckitException(
"buckit.whitelist_functions in {} should be a list",
json_path)
else:
includes_info = None
return PackageInfo(
js["name"],
cell_name.split('/')[-1],
'yarn|{}'.format(js["name"]), package_path, includes_info
)
except Exception as e:
raise BuckitException(
"Could not read property 'buckit.name' or 'name' from "
"json file at {}: {}", json_path, e)
def __init__(self, package_name, json_file, url, commit, tag):
if bool(commit) == bool(tag):
raise BuckitException(
"{}: Either the commit, or the tag must be specified",
json_file)
self.package_name = package_name
self.url = url
self.commit = commit
self.tag = tag
def find_project_root(start_path, node_modules=None):
"""
Starting at start_path, going up, try to find the first directory with
a package.json or .buckconfig, and call that the root of the project
Args:
start_path: The directory to start looking in
node_modules: If provided, the name of the directory that yarn installs
packages to. This can be used if .buckconfig or
package.json do not exist yet
Returns:
The absolute path to the project root
Raises:
Exception: No parent project could be found
"""
terminal = os.path.splitdrive(start_path)[0] or '/'
path = os.path.abspath(start_path)
while path != terminal:
logging.debug("Checking %s for package.json or .buckconfig", path)
package_json = os.path.join(path, PACKAGE_JSON)
package_buckconfig = os.path.join(path, BUCKCONFIG)
package_node_modules = None
if node_modules:
package_node_modules = os.path.join(path, node_modules)
package_path = path
path = os.path.split(path)[0]
if os.path.exists(package_buckconfig):
break
elif os.path.exists(package_json):
try:
package_info = parse_package_info(package_path)
logging.debug(
"Found project %s at %s", package_info.name, package_json
)
break
except Exception:
# If we couldn't parse it, it wasn't meant to be
logging.debug("Could not parse json in %s", package_json)
continue
elif package_node_modules and os.path.exists(package_node_modules):
logging.debug(
"Found node modules directory at %s", package_node_modules)
break
else:
continue
else:
raise BuckitException(
"Could not find a .buckconfig or package.json above {}. Stopped "
"at {}", start_path, path)
logging.debug("{bold}Found project root at %s{clear}", package_path)
return package_path
def install_and_get_files(self, python_settings, pip_command, env):
# TODO: Windows
activate_path = os.path.join(
python_settings.virtualenv_root, 'bin', 'activate'
)
if (not os.path.exists(python_settings.virtualenv_root) or
not os.path.exists(activate_path)):
logging.info(
"Virtualenv at %s does not exist, creating",
python_settings.virtualenv_root
)
readable_check_call(
python_settings.virtualenv_command +
[python_settings.virtualenv_root],
"installing python virtual env",
env=env,
)
package = shlex.quote(
python_settings.pip_package + (python_settings.pip_version or '')
)
command = (
"source bin/activate && {pip} install -I {package} && "
"{pip} show -f {package}"
).format(
pip=pip_command, package=package
)
logging.info(
"Installing %s via pip with %s in %s", package, command,
python_settings.virtualenv_root
)
proc = subprocess.Popen(
args=command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stdin=subprocess.PIPE,
cwd=python_settings.virtualenv_root,
shell=True,
env=env,
)
stdout, stderr = proc.communicate()
if proc.returncode != 0:
logging.error(
"{red}Error installing into virtualenv:{clear}\n"
"stdout: %sstderr: %s\nReturn code %s\n", stdout, stderr,
proc.returncode
)
raise BuckitException(
"Could not install virtualenv at {}",
python_settings.virtualenv_root)
stdout = stdout.decode('utf-8')
return self.parse_pip_output(python_settings, stdout)
def check_hash(self, filename):
file_hash = hashlib.sha256()
with open(filename, 'rb') as fin:
while True:
data = fin.read(self.HASH_BUFFER_SIZE)
if not data:
break
file_hash.update(data)
if file_hash.hexdigest() != self.sha256:
raise BuckitException(
'SHA256 of downloaded file didn\'t match! Expected {}, got {}',
self.sha256, file_hash.hexdigest())
def find_package_paths(
project_root,
node_modules,
package_name='',
already_found=None
):
"""
Find all the packages in a given project root, return information about
each of those packages.
Arguments:
project_root - The root path of the main project
node_modules - The name of the node_modules directory in project_root
package_name - The name of the package underneath node_modules, or
empty if the root package.json should be examined
already_found - A list of package names that have already been
investigated. Used for cycle detection
Returns a tuple of (
dictionary of package names to paths,
set of all of root's direct dependencies,
dictionary of package names to original parsed json
)
"""
already_found = already_found or []
jsons = {}
if package_name:
# Look for cells that have compiled artifacts first
package_root = os.path.join(project_root, node_modules, package_name)
package_json = os.path.join(package_root, PACKAGE_JSON)
else:
package_json = os.path.join(project_root, PACKAGE_JSON)
if os.path.exists(package_json):
with open(package_json, 'r') as fin:
js = json.loads(fin.read())
elif package_name:
logging.debug(
"Could not find a json file at %s for package %s", package_json,
package_name)
return {}, set(), jsons
else:
js = {}
paths = {}
root_deps = set()
# When a package is first added with yarn add, it will not be present
# in package.json in the project root. It will, however, be mentioned in
# environment variables. So, grab from there and make sure that we find
# the package.json in node_modules if we haven't written to the file yet
# If we are just running `buckit buckconfig`, though, make sure we still
# look at all of the root dependencies. Also note that yarn run buckit X
# will populate npm_package_name ,but with a blank string, so we do a
# truthy check
env_deps = {
os.environ['npm_package_name']: os.environ.get('npm_package_version', '*')
} if not package_name and os.environ.get('npm_package_name') else {}
js_deps = js.get('dependencies', {}).copy()
js_deps.update(env_deps)
for dep in js_deps:
if dep in already_found:
raise BuckitException(
'Found a cycle when finding dependencies: {}',
' -> '.join(already_found))
already_found.append(dep)
dep_paths, ignore, dep_jsons = find_package_paths(
project_root, node_modules, dep, already_found
)
paths.update(dep_paths)
jsons.update(dep_jsons)
if not package_name:
root_deps.add(dep)
del already_found[-1]
if package_name:
paths[package_name] = package_root
jsons[package_name] = js
return paths, root_deps, jsons
def populate_cache(self, destination, use_proxy):
raise BuckitException('Not implemented')