Ejemplo n.º 1
0
def load_package_configs():
    package_map = {}

    # Load packages under the pkg/ directory.
    for path in find_packages():
        manifest = json_load(path)
        nuclide_config = manifest.get('nuclide')
        # Skip if not a nuclide package.
        if nuclide_config == None:
            continue
        package_type = nuclide_config.get('packageType')
        test_runner = nuclide_config.get('testRunner')
        disableTests = nuclide_config.get(
            'excludeTestsFromContinuousIntegration', False)
        includeDevDependencies = nuclide_config.get('includeDevDependencies',
                                                    True)

        config = {}
        config['name'] = manifest['name']
        config['repository'] = manifest.get('repository')
        config['version'] = manifest.get('version')
        config['description'] = manifest.get('description')
        config['packageType'] = package_type
        config['isNodePackage'] = package_type == 'Node'
        config['localDependencies'] = {}
        config['packageRootAbsolutePath'] = os.path.dirname(path)
        config['testRunner'] = test_runner
        config['excludeTestsFromContinuousIntegration'] = disableTests
        config['includeDevDependencies'] = includeDevDependencies
        package_map[config['name']] = config

    # Special-case some legacy package-loading code.
    include_legacy_packages = None
    try:
        from fb.legacy_config_loader import legacy_config_loader
        include_legacy_packages = legacy_config_loader
    except Exception as e:
        pass
    if include_legacy_packages:
        include_legacy_packages(package_map)

    # Now that all of the packages have entries in the package_map, use the keys of the package_map
    # to populate the localDependencies array of each config.
    for package_name, package_config in package_map.items():
        package_meta_path = os.path.join(
            package_config['packageRootAbsolutePath'], 'package.json')
        if not os.path.exists(package_meta_path):
            continue

        package_meta = json_load(package_meta_path)
        all_deps = []
        all_deps.extend(package_meta.get('dependencies', {}).keys())
        all_deps.extend(package_meta.get('devDependencies', {}).keys())
        for dependency in all_deps:
            if dependency in package_map:
                package_config['localDependencies'][dependency] = package_map[
                    dependency]

    return package_map
Ejemplo n.º 2
0
def load_package_configs():
    package_map = {}

    # Load packages under the pkg/ directory.
    for path in find_packages():
        manifest = json_load(path)
        nuclide_config = manifest.get('nuclide')
        # Skip if not a nuclide package.
        if nuclide_config == None:
            continue
        package_type = nuclide_config.get('packageType')
        test_runner = nuclide_config.get('testRunner')
        disableTests = nuclide_config.get('excludeTestsFromContinuousIntegration', False)
        includeDevDependencies = nuclide_config.get('includeDevDependencies', True)

        config = {}
        config['name'] = manifest['name']
        config['repository'] = manifest.get('repository')
        config['version'] = manifest.get('version')
        config['description'] = manifest.get('description')
        config['packageType'] = package_type
        config['isNodePackage'] = package_type == 'Node'
        config['localDependencies'] = {}
        config['dependencies'] = manifest.get('dependencies', {})
        config['devDependencies'] = manifest.get('devDependencies', {})
        config['scripts'] = manifest.get('scripts', {})
        config['packageRootAbsolutePath'] = os.path.dirname(path)
        config['testRunner'] = test_runner
        config['excludeTestsFromContinuousIntegration'] = disableTests
        config['includeDevDependencies'] = includeDevDependencies
        package_map[config['name']] = config

    # Special-case some legacy package-loading code.
    include_legacy_packages = None
    try:
        from fb.legacy_config_loader import legacy_config_loader
        include_legacy_packages = legacy_config_loader
    except Exception as e:
        pass
    if include_legacy_packages:
        include_legacy_packages(package_map)

    # Now that all of the packages have entries in the package_map, use the keys of the package_map
    # to populate the localDependencies array of each config.
    for package_name, package_config in package_map.items():
        package_meta_path = os.path.join(package_config['packageRootAbsolutePath'], 'package.json')
        if not os.path.exists(package_meta_path):
            continue

        package_meta = json_load(package_meta_path)
        all_deps = []
        all_deps.extend(package_meta.get('dependencies', {}).keys())
        all_deps.extend(package_meta.get('devDependencies', {}).keys())
        for dependency in all_deps:
            if dependency in package_map:
                package_config['localDependencies'][dependency] = package_map[dependency]

    return package_map
Ejemplo n.º 3
0
def create_placeholder(path_to_package_json):
    metadata = json_load(path_to_package_json)

    # Verify that this represents a Node package.
    nuclide = metadata.get('nuclide', None)
    if not isinstance(nuclide, dict):
        return False
    if nuclide.get('packageType', None) != 'Node':
        return False

    # Verfy that the package has a description.
    if not metadata.get('description', None):
        print('package.json must include a description: %s' % path_to_package_json)
        return False

    # Verify that the npm user is "fb".
    npm_user = fs.cross_platform_check_output(['npm', 'whoami']).rstrip()
    if npm_user != 'fb':
        print(('You must be the "fb" user to publish a Nuclide package to npm, but you were "%s". ' +
                'Find someone who has the appropriate credentials to do the publish for you.') %
                npm_user)
        return False

    # Write out the package.json to a temp directory.
    tmp_dir = tempfile.mkdtemp('npm-publish')
    tmp_package_json = os.path.join(tmp_dir, 'package.json')
    json_dump(metadata, tmp_package_json)

    # Run `npm publish`.
    subprocess.check_call(['npm', 'publish'], cwd=tmp_dir)

    return True
Ejemplo n.º 4
0
def link_dependencys_executable(node_modules_path, dependency_name):
    dependency_root = os.path.join(node_modules_path, dependency_name)
    dependency_config = json_load(os.path.join(dependency_root,
                                               'package.json'))

    # The bin field would ether be a dict or a string. if it's a dict,
    # such as `{ "name": "test", "bin" : { "myapp" : "./cli.js" } }`, we should create a
    # symlink from ./node_modules/test/cli.js to ./node_modules/.bin/myapp.
    # If it's a string, like `{ "name": "test", "bin" : "./cli.js"  }`, then the symlink's name
    # should be name of the package, in this case, it should be ./node_modules/.bin/test .
    bin_config = dependency_config.get('bin')
    if not bin_config:
        return
    elif isinstance(bin_config, dict):
        symlinks_to_create = bin_config
    else:
        symlinks_to_create = {dependency_name: bin_config}

    dot_bin_path = os.path.join(node_modules_path, '.bin')
    if platform_checker.is_windows():
        fs.mkdirs(dot_bin_path)

    for dst_name, relative_src_path in symlinks_to_create.items():
        absolute_dst_path = os.path.join(dot_bin_path, dst_name)
        absolute_src_path = os.path.join(dependency_root, relative_src_path)

        if platform_checker.is_windows():
            shutil.copyfile(absolute_src_path, absolute_dst_path)
        else:
            symlink(absolute_src_path, absolute_dst_path, relative=True)
def link_dependencys_executable(node_modules_path, dependency_name):
    dependency_root = os.path.join(node_modules_path, dependency_name)
    dependency_config = json_load(os.path.join(dependency_root, "package.json"))

    # The bin field would ether be a dict or a string. if it's a dict,
    # such as `{ "name": "test", "bin" : { "myapp" : "./cli.js" } }`, we should create a
    # symlink from ./node_modules/test/cli.js to ./node_modules/.bin/myapp.
    # If it's a string, like `{ "name": "test", "bin" : "./cli.js"  }`, then the symlink's name
    # should be name of the package, in this case, it should be ./node_modules/.bin/test .
    bin_config = dependency_config.get("bin")
    if not bin_config:
        return
    elif isinstance(bin_config, dict):
        symlinks_to_create = bin_config
    else:
        symlinks_to_create = {dependency_name: bin_config}

    dot_bin_path = os.path.join(node_modules_path, ".bin")
    if platform_checker.is_windows():
        fs.mkdirs(dot_bin_path)

    for dst_name, relative_src_path in symlinks_to_create.items():
        absolute_dst_path = os.path.join(dot_bin_path, dst_name)
        absolute_src_path = os.path.join(dependency_root, relative_src_path)

        if platform_checker.is_windows():
            shutil.copyfile(absolute_src_path, absolute_dst_path)
        else:
            symlink(absolute_src_path, absolute_dst_path, relative=True)
Ejemplo n.º 6
0
    def prepublish(self, new_version, atom_semver):
        logging.info('Publishing %s to npm at version %s', self.get_package_name(), new_version)

        # Create temporary directory and copy package into it (without dependencies).
        package = self._config.package_directory
        logging.info('Copying %s to tmpdir', self.get_package_name())
        shutil.copytree(package, self._tmp_package, ignore=shutil.ignore_patterns('node_modules'))

        # Make sure that standard boilerplate files are included in the repo.
        for name, src in self._boilerplate_files.items():
            shutil.copyfile(
                src,
                os.path.join(self._tmp_package, name))

        # Load package.json and rewrite version number within it.
        package_file = os.path.join(self._tmp_package, 'package.json')
        package = json_load(package_file)
        package = update_package_json_versions(self.get_package_name(), package,
            self._config.nuclide_npm_package_names, new_version)

        # Specify the license if it is not already specified.
        if 'license' not in package:
            package['license'] = 'SEE LICENSE IN LICENSE'

        # Write the adjusted package file back to the temporary directory and publish it.
        json_dump(package, package_file)

        # Pre-transpile Babel files, as appropriate.
        self._transpiler.transpile_in_place(self.get_package_name(), self._tmp_package)

        rewrite_shrinkwrap_file(self._tmp_package,
            package, self._config.nuclide_npm_package_names, new_version)
Ejemplo n.º 7
0
    def validate_shrinkwrap_for_package(self, package):
        shrinkwrap_path = os.path.join(package['packageRootAbsolutePath'], 'npm-shrinkwrap.json')
        if not os.path.isfile(shrinkwrap_path):
            # TODO: Enable this once we have shrinkwraps for all packages
            # self.report_error('Expected npm-shrinkwrap.json file at %s not found.', shrinkwrap_path)
            None
        else:
            try:
                shrinkwrap = json_load(shrinkwrap_path)
            except:
                self.report_error('Shrinkwrap file %s not valid JSON.', shrinkwrap_path)
                return

            # Shrinkwrap file must include all top level dependencies
            # And versions of top level dependencies must match
            all_deps = package['dependencies'].copy()
            all_deps.update(package['devDependencies'])
            for dependent_name in shrinkwrap['dependencies'].keys():
                if dependent_name not in all_deps:
                    self.report_error(
                        'Shrinkwrap file %s contains dependency %s which is not in package.json.',
                        shrinkwrap_path, dependent_name)
                elif shrinkwrap['dependencies'][dependent_name]['version'] != all_deps[dependent_name]:
                    self.report_error(
                        'Mismatched version for package %s in shrinkwrap file %s. ' +
                        'package.json contains version %s. Found %s in shrinkwrap file.',
                        dependent_name, shrinkwrap_path, all_deps[dependent_name],
                        shrinkwrap['dependencies'][dependent_name]['version'])

            # All dependencies in the package.json must occur in the Shrinkwrap file.
            for dependent_name in all_deps:
                if dependent_name not in shrinkwrap['dependencies']:
                    self.report_error(
                        'Shrinkwrap file %s missing top level dependency for package %s',
                        shrinkwrap_path, dependent_name)
Ejemplo n.º 8
0
    def get_deps(self, package_json, include_dev_dependencies=False, include_local_dependencies=False):
        '''Return a dependency map: key is package name and value is semver range.'''
        # Although it appears that different versions of a package can be requested by
        # dependencies and devDependencies, it seems as though the one in devDependencies
        # should take priority, and should also be within the range specified by dependencies:
        # http://stackoverflow.com/q/29067071/396304.
        dep_types = ['dependencies']
        if include_dev_dependencies:
            dep_types += ['devDependencies']

        all_deps = {}
        package = json_load(package_json)

        # Apparently both spellings are acceptable:
        bundleDependencies = package.get('bundleDependencies', {})
        bundledDependencies = package.get('bundledDependencies', {})

        for dep_type in dep_types:
            deps = package.get(dep_type, {})
            for dep, version in deps.items():
                if dep in bundleDependencies or dep in bundledDependencies:
                    # There is only one case where we have seen this, which is
                    # https://github.com/goatslacker/testla/tree/717542bfe6a07deab28ffb2da23c989332ae37d6.
                    pass
                elif not self.is_local_dependency(dep) or include_local_dependencies:
                    all_deps[dep] = version

        return all_deps
Ejemplo n.º 9
0
    def prepublish(self, new_version, atom_semver):
        logging.info('Publishing %s to npm at version %s', self.get_package_name(), new_version)

        # Create temporary directory and copy package into it (without dependencies).
        package = self._config.package_directory
        tmp_package = os.path.join(self._tmpdir, self.get_package_name())
        logging.info('Copying %s to tmpdir', self.get_package_name())
        shutil.copytree(package, tmp_package, ignore=shutil.ignore_patterns('node_modules'))

        # Load package.json and rewrite version number within it.
        nil_semver = '0.0.0'
        new_semver = '0.0.%d' % new_version
        package_file = os.path.join(tmp_package, 'package.json')
        package = json_load(package_file)
        if package['version'] != nil_semver:
            raise AssertionError('Local package %s was not at version 0' % self.get_package_name())
        package['version'] = new_semver

        # Update the versions of our local dependencies accordingly.
        for dependency_key in DEPENDENCIES_KEYS:
            if not dependency_key in package:
                continue
            for (dependency, version) in package[dependency_key].items():
                if not self._config.is_nuclide_npm_package(dependency):
                    continue
                if version != nil_semver:
                    raise AssertionError('Local dependency %s in package %s was not at version 0' %
                                         dependency, self.get_package_name())
                package[dependency_key][dependency] = new_semver

        # Write the adjusted package file back to the temporary directory and publish it.
        json_dump(package, package_file)
Ejemplo n.º 10
0
def create_config_for_manifest(path, manifest):
    '''Create a config for a parsed package.json. Returns None if it is not a
    Nuclide package.

    No code in this library should parse a package.json file directly. Instead,
    it should operate on a package config that is created by this method.
    Because we may read extra properties in package.json, such as "customDeps",
    it is critical that all scripts operate on a normalized package config
    rather than a raw package.json.
    '''
    nuclide_config = manifest.get('nuclide')
    # Skip if not a nuclide package.
    if nuclide_config is None:
        return None

    package_type = nuclide_config.get('packageType')
    test_runner = nuclide_config.get('testRunner')
    disableTests = nuclide_config.get(
        'excludeTestsFromContinuousIntegration', False)
    windows_incompatible = nuclide_config.get('windowsIncompatible', False)
    flowCheck = nuclide_config.get('flowCheck', False)

    config = {}
    config['name'] = manifest['name']
    config['repository'] = manifest.get('repository')
    config['version'] = manifest.get('version')
    config['description'] = manifest.get('description')
    config['packageType'] = package_type
    config['isNodePackage'] = package_type == 'Node'
    config['localDependencies'] = {}
    config['dependencies'] = manifest.get('dependencies', {})
    config['devDependencies'] = manifest.get('devDependencies', {})

    # Apparently both spellings are acceptable:
    config['bundleDependencies'] = manifest.get('bundleDependencies', {})
    config['bundledDependencies'] = manifest.get('bundledDependencies', {})

    config['scripts'] = manifest.get('scripts', {})
    config['packageRootAbsolutePath'] = os.path.dirname(path)
    config['testRunner'] = test_runner
    config['excludeTestsFromContinuousIntegration'] = disableTests

    config['flowCheck'] = flowCheck
    config['windowsIncompatible'] = windows_incompatible

    # Check for custom dependencies.
    if 'customDeps' in nuclide_config:
        extra_json = os.path.join(
            os.path.dirname(path), nuclide_config['customDeps'])
        if os.path.exists(extra_json):
            extra_manifest = json_load(extra_json)
            # Track the base dependencies from the published package.json plus
            # the sideloaded "custom" dependencies separately.
            # `config['dependencies']` is the merged set of both.
            config['baseDependencies'] = config['dependencies'].copy()
            config['customDependencies'] = extra_manifest.get(
                'dependencies', {})
            config['dependencies'].update(config['customDependencies'])

    return config
Ejemplo n.º 11
0
    def validate_shrinkwrap_for_package(self, package):
        shrinkwrap_path = os.path.join(package['packageRootAbsolutePath'], 'npm-shrinkwrap.json')
        if not os.path.isfile(shrinkwrap_path):
            # TODO: Enable this once we have shrinkwraps for all packages
            # self.report_error('Expected npm-shrinkwrap.json file at %s not found.', shrinkwrap_path)
            None
        else:
            try:
                shrinkwrap = json_load(shrinkwrap_path)
            except:
                self.report_error('Shrinkwrap file %s not valid JSON.', shrinkwrap_path)
                return

            # Shrinkwrap file must include all top level dependencies
            # And versions of top level dependencies must match
            all_deps = package['dependencies'].copy()
            all_deps.update(package['devDependencies'])
            for dependent_name in shrinkwrap['dependencies'].keys():
                if dependent_name not in all_deps:
                    self.report_error(
                        'Shrinkwrap file %s contains dependency %s which is not in package.json.',
                        shrinkwrap_path, dependent_name)
                elif shrinkwrap['dependencies'][dependent_name]['version'] != all_deps[dependent_name]:
                    self.report_error(
                        'Mismatched version for package %s in shrinkwrap file %s. ' +
                        'package.json contains version %s. Found %s in shrinkwrap file.',
                        dependent_name, shrinkwrap_path, all_deps[dependent_name],
                        shrinkwrap['dependencies'][dependent_name]['version'])

            # All dependencies in the package.json must occur in the Shrinkwrap file.
            for dependent_name in all_deps:
                if dependent_name not in shrinkwrap['dependencies']:
                    self.report_error(
                        'Shrinkwrap file %s missing top level dependency for package %s',
                        shrinkwrap_path, dependent_name)
Ejemplo n.º 12
0
    def _clean_unused_dependencies(self, package_root):
        """Remove unused dependencies for a given package.

        List folders under $package_root/node_modules and compare them to
        $package_root/package.json. If the folder is not a hidden folder (like '.bin') and doesn't
        occur in package.json, run `npm uninstall` to remove it.
        """
        dependencies_root = os.path.join(package_root, 'node_modules')
        if not os.path.exists(dependencies_root):
            return

        package_json_path = os.path.join(package_root, 'package.json')
        meta = json_load(package_json_path)

        # If working with a Nuclide packages, use its custom config builder because it loads info
        # from more places than just the single "package.json".
        if meta.get('nuclide') != None:
            meta = create_config_for_manifest(package_json_path, meta)

        dependencies = set()
        for key in DEPENDENCIES_KEYS:
            dependencies = dependencies.union(set(meta.get(key, {}).keys()))

        for folder in os.listdir(dependencies_root):
            if folder not in dependencies and not folder.startswith('.'):
                logging.info('Uninstall unused dependency %s for %s' % (folder, package_root))
                # Using `npm uninstall` so that the files in .bin will also be removed.
                self._execute(['npm', 'uninstall', folder], package_root)
Ejemplo n.º 13
0
    def get_deps(self,
                 package_json,
                 include_dev_dependencies=False,
                 include_local_dependencies=False):
        '''Return a dependency map: key is package name and value is semver range.'''
        # Although it appears that different versions of a package can be requested by
        # dependencies and devDependencies, it seems as though the one in devDependencies
        # should take priority, and should also be within the range specified by dependencies:
        # http://stackoverflow.com/q/29067071/396304.
        dep_types = ['dependencies']
        if include_dev_dependencies:
            dep_types += ['devDependencies']

        all_deps = {}
        package = json_load(package_json)

        # Apparently both spellings are acceptable:
        bundleDependencies = package.get('bundleDependencies', {})
        bundledDependencies = package.get('bundledDependencies', {})

        for dep_type in dep_types:
            deps = package.get(dep_type, {})
            for dep, version in deps.items():
                if dep in bundleDependencies or dep in bundledDependencies:
                    # There is only one case where we have seen this, which is
                    # https://github.com/goatslacker/testla/tree/717542bfe6a07deab28ffb2da23c989332ae37d6.
                    pass
                elif not self.is_local_dependency(
                        dep) or include_local_dependencies:
                    all_deps[dep] = version

        return all_deps
Ejemplo n.º 14
0
    def _clean_unused_dependencies(self, package_root):
        """Remove unused dependencies for a given package.

        List folders under $package_root/node_modules and compare them to
        $package_root/package.json. If the folder is not a hidden folder (like '.bin') and doesn't
        occur in package.json, run `npm uninstall` to remove it.
        """
        dependencies_root = os.path.join(package_root, 'node_modules')
        if not os.path.exists(dependencies_root):
            return

        package_json_path = os.path.join(package_root, 'package.json')
        meta = json_load(package_json_path)

        # If working with a Nuclide packages, use its custom config builder because it loads info
        # from more places than just the single "package.json".
        if meta.get('nuclide') != None:
            meta = create_config_for_manifest(package_json_path, meta)

        dependencies = set()
        for key in DEPENDENCIES_KEYS:
            dependencies = dependencies.union(set(meta.get(key, {}).keys()))

        for folder in os.listdir(dependencies_root):
            if folder not in dependencies and not folder.startswith('.'):
                logging.info('Uninstall unused dependency %s for %s' %
                             (folder, package_root))
                # Using `npm uninstall` so that the files in .bin will also be removed.
                self._execute(['npm', 'uninstall', folder], package_root)
Ejemplo n.º 15
0
 def _publish_new_versions(self, version, is_dry_run):
     from nuclide_config import NUCLIDE_CONFIG
     atom_semver = json_load(NUCLIDE_CONFIG)['atomVersion']
     logging.info('Publishing packages at new version %d (Atom %s)', version, atom_semver)
     for publisher in self._publishers:
         publisher.prepublish(version, atom_semver)
         if not is_dry_run:
             publisher.publish(version, atom_semver)
Ejemplo n.º 16
0
def create_config_for_manifest(path, manifest):
    '''Create a config for a parsed package.json. Returns None if it is not a
    Nuclide package.

    No code in this library should parse a package.json file directly. Instead,
    it should operate on a package config that is created by this method.
    Because we may read extra properties in package.json, such as "customDeps",
    it is critical that all scripts operate on a normalized package config
    rather than a raw package.json.
    '''
    nuclide_config = manifest.get('nuclide')
    # Skip if not a nuclide package.
    if nuclide_config is None:
        return None

    package_type = nuclide_config.get('packageType')
    test_runner = nuclide_config.get('testRunner')
    disableTests = nuclide_config.get('excludeTestsFromContinuousIntegration',
                                      False)
    flowCheck = nuclide_config.get('flowCheck', False)

    config = {}
    config['name'] = manifest['name']
    config['repository'] = manifest.get('repository')
    config['version'] = manifest.get('version')
    config['description'] = manifest.get('description')
    config['packageType'] = package_type
    config['isNodePackage'] = package_type == 'Node'
    config['localDependencies'] = {}
    config['dependencies'] = manifest.get('dependencies', {})
    config['devDependencies'] = manifest.get('devDependencies', {})

    # Apparently both spellings are acceptable:
    config['bundleDependencies'] = manifest.get('bundleDependencies', {})
    config['bundledDependencies'] = manifest.get('bundledDependencies', {})

    config['scripts'] = manifest.get('scripts', {})
    config['packageRootAbsolutePath'] = os.path.dirname(path)
    config['testRunner'] = test_runner
    config['excludeTestsFromContinuousIntegration'] = disableTests

    config['flowCheck'] = flowCheck

    # Check for custom dependencies.
    if 'customDeps' in nuclide_config:
        extra_json = os.path.join(os.path.dirname(path),
                                  nuclide_config['customDeps'])
        if os.path.exists(extra_json):
            extra_manifest = json_load(extra_json)
            # Track the base dependencies from the published package.json plus
            # the sideloaded "custom" dependencies separately.
            # `config['dependencies']` is the merged set of both.
            config['baseDependencies'] = config['dependencies'].copy()
            config['customDependencies'] = extra_manifest.get(
                'dependencies', {})
            config['dependencies'].update(config['customDependencies'])

    return config
Ejemplo n.º 17
0
 def _publish_new_versions(self, version, is_dry_run):
     from nuclide_config import NUCLIDE_CONFIG
     atom_semver = json_load(NUCLIDE_CONFIG)['atomVersion']
     logging.info('Publishing packages at new version %d (Atom %s)',
                  version, atom_semver)
     for publisher in self._publishers:
         publisher.prepublish(version, atom_semver)
         if not is_dry_run:
             publisher.publish(version, atom_semver)
Ejemplo n.º 18
0
def create_placeholder(path_to_package_json, apm_repository_helper):
    metadata = json_load(path_to_package_json)

    # Verify that this represents an Atom package.
    nuclide = metadata.get('nuclide', None)
    if not isinstance(nuclide, dict):
        return False
    if nuclide.get('packageType', None) != 'Atom' or nuclide.get('testRunner', None) != 'apm':
        return False

    # Create or checkout the repo for the package under facebooknuclideapm.
    package_name = metadata['name']
    repo = tempfile.mkdtemp(package_name)
    apm_repository_helper.checkout_apm_repo(package_name, repo, create_if_missing=True)
    git = apm_repository_helper.git

    # Verify the package.json file has been checked in. Commit it if it has not.
    package_json_in_repo = os.path.join(repo, 'package.json')
    if not os.path.isfile(package_json_in_repo):
        metadata['repository'] = 'https://github.com/facebooknuclideapm/%s' % package_name
        json_dump(metadata, package_json_in_repo)
        git.commit_all(repo, 'Commit package.json file.')
        git.push_to_master(repo)

    # Verify the README.md has been checked in. Commit it if it has not.
    path_to_readme = os.path.join(repo, 'README.md')
    if not os.path.isfile(path_to_readme):
        with open(path_to_readme, 'w') as f:
            f.write('Placeholder.')
        git.commit_all(repo, 'Commit README.md file.')
        git.push_to_master(repo)

    # Create and push tag v0.0.0.
    tag_name = 'v0.0.0'
    tag_exists = False
    try:
        git.add_tag(repo, tag_name, 'Atom package %s.' % tag_name)
    except subprocess.CalledProcessError:
        tag_exists = True
    if not tag_exists:
        git.push_tag(repo, tag_name)
        git.push_to_master(repo)

    # Run `apm publish` to claim the namespace.
    try:
        subprocess.check_call(['apm', 'publish', '--tag', tag_name], cwd=repo)
    except subprocess.CalledProcessError:
        # `apm publish` may complain about some things, but at a minimum, the namespace should be
        # claimed.
        pass

    # Verify whether the publish succeeded.
    try:
        subprocess.check_call(['apm', 'view', package_name], cwd=repo)
        return True
    except subprocess.CalledProcessError:
        return False
Ejemplo n.º 19
0
def process_package(pkg, copy_local_dependencies, queue, package_manager,
                    npm_directory):
    logging.info('OfflineInstaller is installing %s', pkg.name)
    package_json = pkg.package_json
    all_deps = package_manager.get_deps(package_json,
                                        pkg.include_dev_dependencies,
                                        include_local_dependencies=True)
    if not all_deps:
        return

    package_root = os.path.dirname(package_json)
    node_modules = os.path.join(package_root, 'node_modules')
    bin_dir = os.path.join(node_modules, '.bin')
    mkdirs(node_modules)
    for name, version in all_deps.items():
        package_dir = os.path.join(node_modules, name)
        if package_manager.is_local_dependency(name):
            if copy_local_dependencies:
                # A packaging tool may want the option to copy rather than symlink dependencies.
                shutil.copytree(package_manager.get_local_package_root(name),
                                package_dir)
            else:
                # Prefer local symlink if it is an option.
                symlink(package_manager.get_local_package_root(name),
                        package_dir)
        # Install the dependency at node_modules/pkg_name.
        # Note that if there is a compatible version in a parent node_modules,
        # then you should not install it again in order to save space
        # (and in some cases, to avoid cycles).
        elif not has_ancestor_with_dep(name, version, node_modules):
            # TODO: If the package.json has a preinstall step, it should be run.
            install_package(name, version, package_dir, npm_directory)

            # Add the package.json for the dependency to the queue.
            pkg_to_install = PackageNeedsDepsInstalled(
                name,
                os.path.join(package_dir, 'package.json'),
                include_dev_dependencies=False)
            queue.appendleft(pkg_to_install)
        else:
            # Unclear whether .bin should still get installed in this case. If so,
            # has_ancestor_with_dep() should be changed to return the path to the ancestor.
            continue

        # If the dependency's package.json has bin entries, then they need to be
        # symlinked to the dependent package's node_modules/.bin directory.
        package_info = json_load(os.path.join(package_dir, 'package.json'))
        bin = package_info.get('bin', None)
        if isinstance(bin, str):
            bin_command = bin
            bin = {}
            bin[package_info['name']] = bin_command
        if isinstance(bin, dict) and bin:
            mkdirs(bin_dir)
            for script_name, local_path in bin.items():
                symlink(os.path.join(package_dir, local_path),
                        os.path.join(bin_dir, script_name))
def rewrite_shrinkwrap_file(package_dir, package_name, dependent_packages, new_version):
    shrinkwrap_file = os.path.join(package_dir, 'npm-shrinkwrap.json')
    # TODO(peterhal): Remove this test once we have shrinkwraps in place for all packages
    if os.path.isfile(shrinkwrap_file):
        shrinkwrap = json_load(shrinkwrap_file)
        shrinkwrap = update_shrinkwrap_json_versions(package_name, shrinkwrap,
            dependent_packages, new_version)
        # Write the adjusted shrinkwrap file back to the temporary directory and publish it.
        json_dump(shrinkwrap, shrinkwrap_file)
Ejemplo n.º 21
0
 def _process_package_dir(self, src_path, package_to_version_set, include_dev_dependencies):
     shrinkwrap_file = os.path.join(src_path, 'npm-shrinkwrap.json')
     if os.path.isfile(shrinkwrap_file):
         shrinkwrap_json = json_load(shrinkwrap_file)
         self._process_shrinkwrap_json(shrinkwrap_json, package_to_version_set)
     else:
         package_json = os.path.join(src_path, 'package.json')
         self._process_package_json(package_json,
                                    package_to_version_set,
                                    include_dev_dependencies)
Ejemplo n.º 22
0
 def _process_package_dir(self, src_path, package_to_version_set,
                          include_dev_dependencies):
     shrinkwrap_file = os.path.join(src_path, 'npm-shrinkwrap.json')
     if os.path.isfile(shrinkwrap_file):
         shrinkwrap_json = json_load(shrinkwrap_file)
         self._process_shrinkwrap_json(shrinkwrap_json,
                                       package_to_version_set)
     else:
         package_json = os.path.join(src_path, 'package.json')
         self._process_package_json(package_json, package_to_version_set,
                                    include_dev_dependencies)
Ejemplo n.º 23
0
def rewrite_shrinkwrap_file(package_dir, package_name, dependent_packages,
                            new_version):
    shrinkwrap_file = os.path.join(package_dir, 'npm-shrinkwrap.json')
    # TODO(peterhal): Remove this test once we have shrinkwraps in place for all packages
    if os.path.isfile(shrinkwrap_file):
        shrinkwrap = json_load(shrinkwrap_file)
        shrinkwrap = update_shrinkwrap_json_versions(package_name, shrinkwrap,
                                                     dependent_packages,
                                                     new_version)
        # Write the adjusted shrinkwrap file back to the temporary directory and publish it.
        json_dump(shrinkwrap, shrinkwrap_file)
Ejemplo n.º 24
0
    def create_transpiler(path_to_nuclide_repo, transpile_script):
        '''Creates a new Transpiler object.

        Args:
            path_to_nuclide_repo: Path to directory that contains the pkg/ and scripts/
                subdirectories. It is used to locate the config files that determine which Babel
                files should not get transpiled.
            transpile_script: Path to script that takes a file path for a Babel file and returns
                the transpiled output (using the same Babel transpilation options as Atom).
        '''

        # Keys are package names. (Note these could be Node or Atom packages.)
        # Values are arrays of relative paths under the package that identify paths that should not
        # be transpiled.
        exclude_from_transpilation = {}
        service_framework_v3_config = 'pkg/nuclide/server/services-3.json'
        config_files = [
            # Service framework v2 configuration.
            'pkg/nuclide/server/fb/custom-services-config.json',
            'pkg/nuclide/server/services-config.json',
            # Service framework v3 configuration.
            service_framework_v3_config,
        ]
        for config_file in config_files:
            abs_config_file_path = os.path.join(path_to_nuclide_repo,
                                                config_file)
            if not os.path.isfile(abs_config_file_path):
                continue

            entries = json_helpers.json_load(abs_config_file_path)
            for entry in entries:
                if config_file == service_framework_v3_config:
                    # For service framework v3 config entry, skip transpile the implementation
                    # file only if the definition file is omitted.
                    if entry.get('definition'):
                        continue
                    exclude_file = entry['implementation']
                else:
                    if not entry['useServiceFramework']:
                        continue
                    # For service framework v2 config entry, skip transpile the definition file.
                    exclude_file = entry['definition']

                index = exclude_file.index('/')
                package_name = exclude_file[:index]
                relative_path = exclude_file[index + 1:]
                if package_name not in exclude_from_transpilation:
                    paths = []
                    exclude_from_transpilation[package_name] = paths
                else:
                    paths = exclude_from_transpilation[package_name]
                paths.append(relative_path)

        return Transpiler(exclude_from_transpilation, transpile_script)
Ejemplo n.º 25
0
    def validate_babelrc(self, package):
        # See https://phabricator.fb.com/D2301649 for details on why this exists.
        babelrc_path = os.path.join(package['packageRootAbsolutePath'], '.babelrc')
        if not os.path.isfile(babelrc_path):
            self.report_error('Expected .babelrc file at %s not found.', babelrc_path)
            return

        babel_options = json_load(babelrc_path)
        expected_options = {'breakConfig': True}
        if not babel_options == expected_options:
            self.report_error('.babelrc file %s had options %s but expected %s' %
                (babelrc_path, babel_options, expected_options))
Ejemplo n.º 26
0
    def create_transpiler(path_to_nuclide_repo, transpile_script):
        '''Creates a new Transpiler object.

        Args:
            path_to_nuclide_repo: Path to directory that contains the pkg/ and scripts/
                subdirectories. It is used to locate the config files that determine which Babel
                files should not get transpiled.
            transpile_script: Path to script that takes a file path for a Babel file and returns
                the transpiled output (using the same Babel transpilation options as Atom).
        '''

        # Keys are package names. (Note these could be Node or Atom packages.)
        # Values are arrays of relative paths under the package that identify paths that should not
        # be transpiled.
        exclude_from_transpilation = {}
        service_framework_v3_config = 'pkg/nuclide/server/services-3.json'
        config_files = [
            # Service framework v2 configuration.
            'pkg/nuclide/server/fb/custom-services-config.json',
            'pkg/nuclide/server/services-config.json',
            # Service framework v3 configuration.
            service_framework_v3_config,
        ]
        for config_file in config_files:
            abs_config_file_path = os.path.join(path_to_nuclide_repo, config_file)
            if not os.path.isfile(abs_config_file_path):
                continue

            entries = json_helpers.json_load(abs_config_file_path)
            for entry in entries:
                if config_file == service_framework_v3_config:
                    # For service framework v3 config entry, skip transpile the implementation
                    # file only if the definition file is omitted.
                    if entry.get('definition'):
                        continue
                    exclude_file = entry['implementation']
                else:
                    if not entry['useServiceFramework']:
                        continue
                    # For service framework v2 config entry, skip transpile the definition file.
                    exclude_file = entry['definition']

                index = exclude_file.index('/')
                package_name = exclude_file[:index]
                relative_path = exclude_file[index + 1:]
                if package_name not in exclude_from_transpilation:
                    paths = []
                    exclude_from_transpilation[package_name] = paths
                else:
                    paths = exclude_from_transpilation[package_name]
                paths.append(relative_path)

        return Transpiler(exclude_from_transpilation, transpile_script)
Ejemplo n.º 27
0
    def validate_babelrc(self, package):
        # See https://phabricator.fb.com/D2301649 for details on why this exists.
        babelrc_path = os.path.join(package['packageRootAbsolutePath'], '.babelrc')
        if not os.path.isfile(babelrc_path):
            self.report_error('Expected .babelrc file at %s not found.', babelrc_path)
            return

        babel_options = json_load(babelrc_path)
        expected_options = {'breakConfig': True}
        if not babel_options == expected_options:
            self.report_error('.babelrc file %s had options %s but expected %s' %
                (babelrc_path, babel_options, expected_options))
Ejemplo n.º 28
0
def process_package(pkg, copy_local_dependencies, queue, package_manager, npm_directory):
    logging.info('OfflineInstaller is installing %s', pkg.name)
    package_json = pkg.package_json
    all_deps = package_manager.get_deps(package_json,
                                        pkg.include_dev_dependencies,
                                        include_local_dependencies=True)
    if not all_deps:
        return

    package_root = os.path.dirname(package_json)
    node_modules = os.path.join(package_root, 'node_modules')
    bin_dir = os.path.join(node_modules, '.bin')
    mkdirs(node_modules)
    for name, version in all_deps.items():
        package_dir = os.path.join(node_modules, name)
        if package_manager.is_local_dependency(name):
            if copy_local_dependencies:
                # A packaging tool may want the option to copy rather than symlink dependencies.
                shutil.copytree(package_manager.get_local_package_root(name), package_dir)
            else:
                # Prefer local symlink if it is an option.
                symlink(package_manager.get_local_package_root(name), package_dir)
        # Install the dependency at node_modules/pkg_name.
        # Note that if there is a compatible version in a parent node_modules,
        # then you should not install it again in order to save space
        # (and in some cases, to avoid cycles).
        elif not has_ancestor_with_dep(name, version, node_modules):
            # TODO: If the package.json has a preinstall step, it should be run.
            install_package(name, version, package_dir, npm_directory)

            # Add the package.json for the dependency to the queue.
            pkg_to_install = PackageNeedsDepsInstalled(name,
                                                       os.path.join(package_dir, 'package.json'),
                                                       include_dev_dependencies=False)
            queue.appendleft(pkg_to_install)
        else:
            # Unclear whether .bin should still get installed in this case. If so,
            # has_ancestor_with_dep() should be changed to return the path to the ancestor.
            continue

        # If the dependency's package.json has bin entries, then they need to be
        # symlinked to the dependent package's node_modules/.bin directory.
        package_info = json_load(os.path.join(package_dir, 'package.json'))
        bin = package_info.get('bin', None)
        if isinstance(bin, str):
            bin_command = bin
            bin = {}
            bin[package_info['name']] = bin_command
        if isinstance(bin, dict) and bin:
            mkdirs(bin_dir)
            for script_name, local_path in bin.items():
                symlink(os.path.join(package_dir, local_path), os.path.join(bin_dir, script_name))
Ejemplo n.º 29
0
    def _publish_new_versions(self, version, is_dry_run):
        from nuclide_config import NUCLIDE_CONFIG
        atom_semver = json_load(NUCLIDE_CONFIG)['atomVersion']
        logging.info('Publishing packages at new version %d (Atom %s)', version, atom_semver)
        for publisher in self._publishers:
            if publisher.is_already_published(version) and not is_dry_run:
                logging.info('Skipping %[email protected].%s: already published.' %
                    (publisher.get_package_name(), version))
                continue

            publisher.prepublish(version, atom_semver)
            if not is_dry_run:
                publisher.publish(version, atom_semver)
Ejemplo n.º 30
0
def has_ancestor_with_dep(name, version, node_modules):
    candidate_dir = os.path.join(node_modules, name)
    if os.path.isdir(candidate_dir):
        package_json = os.path.join(candidate_dir, 'package.json')
        discovered_version = json_load(package_json)['version']
        if find_version_in_range([discovered_version], version):
            return True
    # Look upwards to the next node_modules, if present.
    parent_node_modules = os.path.normpath(os.path.join(node_modules, '../../'))
    if os.path.basename(parent_node_modules) == 'node_modules':
        return has_ancestor_with_dep(name, version, parent_node_modules)
    else:
        return False
Ejemplo n.º 31
0
    def _publish_new_versions(self, version, is_dry_run):
        from nuclide_config import NUCLIDE_CONFIG
        atom_semver = json_load(NUCLIDE_CONFIG)['atomVersion']
        logging.info('Publishing packages at new version %d (Atom %s)',
                     version, atom_semver)
        for publisher in self._publishers:
            if publisher.is_already_published(version) and not is_dry_run:
                logging.info('Skipping %[email protected].%s: already published.' %
                             (publisher.get_package_name(), version))
                continue

            publisher.prepublish(version, atom_semver)
            if not is_dry_run:
                publisher.publish(version, atom_semver)
Ejemplo n.º 32
0
    def get_deps(self,
                 package_json,
                 include_dev_dependencies=False,
                 include_local_dependencies=False):
        '''Return a dependency map: key is package name and value is semver range.'''
        # Although it appears that different versions of a package can be requested by
        # dependencies and devDependencies, it seems as though the one in devDependencies
        # should take priority, and should also be within the range specified by dependencies:
        # http://stackoverflow.com/q/29067071/396304.
        dep_types = ['dependencies']
        if include_dev_dependencies:
            dep_types += ['devDependencies']

        all_deps = {}
        config = create_config_for_package(package_json)
        if not config:
            # config is None: this must be a transitive dependency of a Nuclide package.
            package = json_load(package_json)
            config = {
                'dependencies': package.get('dependencies', {}),
                'devDependencies': package.get('devDependencies', {}),
                'bundleDependencies': package.get('bundleDependencies', {}),
                'bundledDependencies': package.get('bundledDependencies', {}),
                'optionalDependencies': package.get('optionalDependencies',
                                                    {}),
            }
            # optionalDependencies override dependencies and do not get installed.
            # So we remove them from dependencies.
            for dep in config['optionalDependencies'].keys():
                config['dependencies'].pop(dep, None)

        # Apparently both spellings are acceptable:
        bundleDependencies = config['bundleDependencies']
        bundledDependencies = config['bundledDependencies']

        for dep_type in dep_types:
            deps = config[dep_type]
            for dep, version in deps.items():
                if dep in bundleDependencies or dep in bundledDependencies:
                    # There is only one case where we have seen this, which is
                    # https://github.com/goatslacker/testla/tree/717542bfe6a07deab28ffb2da23c989332ae37d6.
                    pass
                elif not self.is_local_dependency(
                        dep) or include_local_dependencies:
                    all_deps[dep] = version

        return all_deps
Ejemplo n.º 33
0
        def process_packages(packages, is_npm):
            for package_json in packages:
                package_name = json_load(package_json)['name']
                if is_npm:
                    nuclide_npm_packages.add(package_name)
                else:
                    nuclide_apm_packages.add(package_name)

                config = AbstractPublisherConfig(
                    package_name,
                    os.path.dirname(package_json),
                    nuclide_npm_packages,
                    nuclide_apm_packages)
                if is_npm:
                    publisher = NpmPublisher(config, npm, master_tmpdir, transpiler, boilerplate_files)
                else:
                    publisher = ApmPublisher(config, apm, master_tmpdir, transpiler, boilerplate_files, git, github_access_token)
                publishers.append(publisher)
Ejemplo n.º 34
0
        def process_packages(packages, is_npm):
            for package_json in packages:
                package_name = json_load(package_json)['name']
                if is_npm:
                    nuclide_npm_packages.add(package_name)
                else:
                    nuclide_apm_packages.add(package_name)

                config = AbstractPublisherConfig(package_name,
                                                 os.path.dirname(package_json),
                                                 nuclide_npm_packages,
                                                 nuclide_apm_packages)
                if is_npm:
                    publisher = NpmPublisher(config, npm, master_tmpdir)
                else:
                    publisher = ApmPublisher(config, apm, master_tmpdir, git,
                                             github_access_token)
                publishers.append(publisher)
Ejemplo n.º 35
0
    def get_deps(self, package_json, include_dev_dependencies=False, include_local_dependencies=False):
        '''Return a dependency map: key is package name and value is semver range.'''
        # Although it appears that different versions of a package can be requested by
        # dependencies and devDependencies, it seems as though the one in devDependencies
        # should take priority, and should also be within the range specified by dependencies:
        # http://stackoverflow.com/q/29067071/396304.
        dep_types = ['dependencies']
        if include_dev_dependencies:
            dep_types += ['devDependencies']

        all_deps = {}
        config = create_config_for_package(package_json)
        if not config:
            # config is None: this must be a transitive dependency of a Nuclide package.
            package = json_load(package_json)
            config = {
              'dependencies': package.get('dependencies', {}),
              'devDependencies': package.get('devDependencies', {}),
              'bundleDependencies': package.get('bundleDependencies', {}),
              'bundledDependencies': package.get('bundledDependencies', {}),
              'optionalDependencies': package.get('optionalDependencies', {}),
            }
            # optionalDependencies override dependencies and do not get installed.
            # So we remove them from dependencies.
            for dep in config['optionalDependencies'].keys():
                config['dependencies'].pop(dep, None)

        # Apparently both spellings are acceptable:
        bundleDependencies = config['bundleDependencies']
        bundledDependencies = config['bundledDependencies']

        for dep_type in dep_types:
            deps = config[dep_type]
            for dep, version in deps.items():
                if dep in bundleDependencies or dep in bundledDependencies:
                    # There is only one case where we have seen this, which is
                    # https://github.com/goatslacker/testla/tree/717542bfe6a07deab28ffb2da23c989332ae37d6.
                    pass
                elif not self.is_local_dependency(dep) or include_local_dependencies:
                    all_deps[dep] = version

        return all_deps
Ejemplo n.º 36
0
    def create_transpiler(path_to_nuclide_repo, transpile_script):
        '''Creates a new Transpiler object.

        Args:
            path_to_nuclide_repo: Path to directory that contains the pkg/ and scripts/
                subdirectories. It is used to locate the config files that determine which Babel
                files should not get transpiled.
            transpile_script: Path to script that takes a file path for a Babel file and returns
                the transpiled output (using the same Babel transpilation options as Atom).
        '''

        # Keys are package names. (Note these could be Node or Atom packages.)
        # Values are arrays of relative paths under the package that identify paths that should not
        # be transpiled.
        exclude_from_transpilation = {}
        config_files = [
            'pkg/nuclide/server/fb/custom-services-config.json',
            'pkg/nuclide/server/services-config.json',
        ]
        for config_file in config_files:
            config_file = os.path.join(path_to_nuclide_repo, config_file)
            if not os.path.isfile(config_file):
                continue

            entries = json_helpers.json_load(config_file)
            for entry in entries:
                if not entry['useServiceFramework']:
                    continue

                definition = entry['definition']
                index = definition.index('/')
                package_name = definition[:index]
                relative_path = definition[index + 1:]
                if package_name not in exclude_from_transpilation:
                    paths = []
                    exclude_from_transpilation[package_name] = paths
                else:
                    paths = exclude_from_transpilation[package_name]
                paths.append(relative_path)

        return Transpiler(exclude_from_transpilation, transpile_script)
Ejemplo n.º 37
0
    def create_transpiler(path_to_nuclide_repo, transpile_script):
        '''Creates a new Transpiler object.

        Args:
            path_to_nuclide_repo: Path to directory that contains the pkg/ and scripts/
                subdirectories. It is used to locate the config files that determine which Babel
                files should not get transpiled.
            transpile_script: Path to script that takes a file path for a Babel file and returns
                the transpiled output (using the same Babel transpilation options as Atom).
        '''

        # Keys are package names. (Note these could be Node or Atom packages.)
        # Values are arrays of relative paths under the package that identify paths that should not
        # be transpiled.
        exclude_from_transpilation = {}
        config_files = [
            'pkg/nuclide/server/fb/custom-services-config.json',
            'pkg/nuclide/server/services-config.json',
        ]
        for config_file in config_files:
            config_file = os.path.join(path_to_nuclide_repo, config_file)
            if not os.path.isfile(config_file):
                continue

            entries = json_helpers.json_load(config_file)
            for entry in entries:
                if not entry['useServiceFramework']:
                    continue

                definition = entry['definition']
                index = definition.index('/')
                package_name = definition[:index]
                relative_path = definition[index + 1:]
                if package_name not in exclude_from_transpilation:
                    paths = []
                    exclude_from_transpilation[package_name] = paths
                else:
                    paths = exclude_from_transpilation[package_name]
                paths.append(relative_path)

        return Transpiler(exclude_from_transpilation, transpile_script)
Ejemplo n.º 38
0
    def prepublish(self, new_version, atom_semver):
        logging.info('Publishing %s to npm at version %s',
                     self.get_package_name(), new_version)

        # Create temporary directory and copy package into it (without dependencies).
        package = self._config.package_directory
        logging.info('Copying %s to tmpdir', self.get_package_name())
        shutil.copytree(package,
                        self._tmp_package,
                        ignore=shutil.ignore_patterns('node_modules'))

        # Make sure that standard boilerplate files are included in the repo.
        for name, src in self._boilerplate_files.items():
            shutil.copyfile(src, os.path.join(self._tmp_package, name))

        # Load package.json and rewrite version number within it.
        package_file = os.path.join(self._tmp_package, 'package.json')
        package = json_load(package_file)
        package = update_package_json_versions(
            self.get_package_name(), package,
            self._config.nuclide_npm_package_names, new_version)

        # Delete "_atomModuleCache" field from package.json.
        # TODO (chenshen): delete following line once '_atomModuleCache' is not fake.
        if '_atomModuleCache' in package:
            del package['_atomModuleCache']

        # Specify the license if it is not already specified.
        if 'license' not in package:
            package['license'] = 'SEE LICENSE IN LICENSE'

        # Write the adjusted package file back to the temporary directory and publish it.
        json_dump(package, package_file)

        # Pre-transpile Babel files, as appropriate.
        self._transpiler.transpile_in_place(self.get_package_name(),
                                            self._tmp_package)

        rewrite_shrinkwrap_file(self._tmp_package, package,
                                self._config.nuclide_npm_package_names,
                                new_version)
Ejemplo n.º 39
0
    def _clean_unused_dependencies(self, package_root):
        """Remove unused dependencies for a given package.

        List folders under $pacakge_root/node_modules and compare them to
        $package_root/package.json. If the folder is not a hidden folder (like '.bin') and doesn't
        occur in package.json, run `npm uninstall` to remove it.
        """
        dependencies_root = os.path.join(package_root, 'node_modules')
        if not os.path.exists(dependencies_root):
            return

        meta = json_load(os.path.join(package_root, 'package.json'))
        dependencies = set()
        for key in DEPENDENCIES_KEYS:
            dependencies = dependencies.union(set(meta.get(key, {}).keys()))

        for folder in os.listdir(dependencies_root):
            if folder not in dependencies and not folder.startswith('.'):
                logging.info('Uninstall unused dependency %s for %s' % (folder, package_root))
                # Using `npm uninstall` so that the files in .bin will also be removed.
                self._execute(['npm', 'uninstall', folder], package_root)
Ejemplo n.º 40
0
    def _clean_unused_dependencies(self, package_root):
        """Remove unused dependencies for a given package.

        List folders under $pacakge_root/node_modules and compare them to
        $package_root/package.json. If the folder is not a hidden folder (like '.bin') and doesn't
        occur in package.json, run `npm uninstall` to remove it.
        """
        dependencies_root = os.path.join(package_root, 'node_modules')
        if not os.path.exists(dependencies_root):
            return

        meta = json_load(os.path.join(package_root, 'package.json'))
        dependencies = set()
        for key in DEPENDENCIES_KEYS:
            dependencies = dependencies.union(set(meta.get(key, {}).keys()))

        for folder in os.listdir(dependencies_root):
            if folder not in dependencies and not folder.startswith('.'):
                logging.info('Uninstall unused dependency %s for %s' % (folder, package_root))
                # Using `npm uninstall` so that the files in .bin will also be removed.
                self._execute(['npm', 'uninstall', folder], package_root)
Ejemplo n.º 41
0
    def prepublish(self, new_version, atom_semver):
        logging.info('Publishing %s to npm at version %s',
                     self.get_package_name(), new_version)

        # Create temporary directory and copy package into it (without dependencies).
        package = self._config.package_directory
        logging.info('Copying %s to tmpdir', self.get_package_name())
        shutil.copytree(package,
                        self._tmp_package,
                        ignore=shutil.ignore_patterns('node_modules'))

        # Make sure that standard boilerplate files are included in the repo.
        for name, src in self._boilerplate_files.items():
            shutil.copyfile(src, os.path.join(self._tmp_package, name))

        # Load package.json and rewrite version number within it.
        nil_semver = '0.0.0'
        new_semver = '0.0.%d' % new_version
        package_file = os.path.join(self._tmp_package, 'package.json')
        package = json_load(package_file)
        if package['version'] != nil_semver:
            raise AssertionError('Local package %s was not at version 0' %
                                 self.get_package_name())
        package['version'] = new_semver

        # Update the versions of our local dependencies accordingly.
        for dependency_key in DEPENDENCIES_KEYS:
            if not dependency_key in package:
                continue
            for (dependency, version) in package[dependency_key].items():
                if not self._config.is_nuclide_npm_package(dependency):
                    continue
                if version != nil_semver:
                    raise AssertionError(
                        'Local dependency %s in package %s was not at version 0'
                        % dependency, self.get_package_name())
                package[dependency_key][dependency] = new_semver

        # Write the adjusted package file back to the temporary directory and publish it.
        json_dump(package, package_file)
Ejemplo n.º 42
0
    def prepublish(self, new_version, atom_semver):
        self._repo = self._checkout_apm_repo()

        logging.info('Publishing %s to apm at version %s',
                     self.get_package_name(), new_version)

        # Clean repo out, leaving .git metadata in place.
        logging.info('Cleaning repo for package %s', self.get_package_name())
        self.clean_repo()

        # Copy files from package into repo.
        package = self._config.package_directory
        logging.info('Copying package %s', self.get_package_name())
        self.copy_into_repo(package)

        # Make sure that standard boilerplate files are included in the repo.
        for name, src in self._boilerplate_files.items():
            shutil.copyfile(src, os.path.join(self._repo, name))

        # Load package.json and rewrite version number within it.
        package_file = os.path.join(self._repo, 'package.json')
        package = json_load(package_file)
        package = update_package_json_versions(
            self.get_package_name(), package,
            self._config.nuclide_npm_package_names, new_version)

        # Specify the license if it is not already specified.
        if 'license' not in package:
            package['license'] = 'SEE LICENSE IN LICENSE'

        # Update the version of the Atom engine required.
        package['engines'] = {'atom': '>=%s' % atom_semver}

        # Rewrite the repository URL.
        repository = 'https://github.com/facebooknuclideapm/%s' % self.get_package_name(
        )
        package['repository'] = repository

        # Write the adjusted package file back to the temporary directory and publish it.
        json_dump(package, package_file)

        rewrite_shrinkwrap_file(self._repo, package,
                                self._config.nuclide_npm_package_names,
                                new_version)

        # Add a boilerplate .gitignore file if the package does not already have one.
        path_to_gitignore = os.path.join(self._repo, '.gitignore')
        if not os.path.exists(path_to_gitignore):
            with open(path_to_gitignore, 'w') as f:
                f.write(DEFAULT_GITIGNORE)

        # Prefix the README.md with information about the proper repository.
        path_to_readme = os.path.join(self._repo, 'README.md')
        if os.path.exists(path_to_readme):
            with open(path_to_readme, 'r') as f:
                readme_contents = README_PREFIX + f.read()
        else:
            readme_contents = README_PREFIX
        with open(path_to_readme, 'w') as f:
            f.write(readme_contents)

        # Write out the packages to install for the nuclide-installer package.
        if self.get_package_name() == 'nuclide-installer':
            from publishers.nuclide_installer_config import generate_config
            installer_config_json = generate_config(
                package['version'], self._config.apm_package_names)
            with open(os.path.join(self._repo, 'lib', 'config.json'),
                      'w') as f:
                f.write(installer_config_json)

        # Pre-transpile Babel files, as appropriate.
        self._transpiler.transpile_in_place(self.get_package_name(),
                                            self._repo)
Ejemplo n.º 43
0
    def prepublish(self, new_version, atom_semver):
        self._repo = self._checkout_apm_repo()

        logging.info('Publishing %s to apm at version %s',
                     self.get_package_name(), new_version)

        # Clean repo out, leaving .git metadata in place.
        logging.info('Cleaning repo for package %s', self.get_package_name())
        self.clean_repo()

        # Copy files from package into repo.
        package = self._config.package_directory
        logging.info('Copying package %s', self.get_package_name())
        self.copy_into_repo(package)

        # Load package.json and rewrite version number within it.
        # TODO (jpearce): reconcile with very similar npm code
        nil_semver = '0.0.0'
        new_semver = '0.0.%d' % new_version
        package_file = os.path.join(self._repo, 'package.json')
        package = json_load(package_file)
        if package['version'] != nil_semver:
            raise AssertionError('Local package %s was not at version 0' %
                                 self.get_package_name())
        package['version'] = new_semver

        # Update the versions of our local dependencies accordingly.
        for dependency_key in DEPENDENCIES_KEYS:
            if not dependency_key in package:
                continue
            for (dependency, version) in package[dependency_key].items():
                if not self._config.is_nuclide_npm_package(dependency):
                    continue
                if version != nil_semver:
                    raise AssertionError(
                        'Local dependency %s in package %s was not at version 0'
                        % dependency, self.get_package_name())
                package[dependency_key][dependency] = new_semver

        # Update the version of the Atom engine required.
        package['engines'] = {'atom': '>=%s' % atom_semver}

        # Rewrite the repository URL.
        repository = 'https://github.com/facebooknuclideapm/%s' % self.get_package_name(
        )
        package['repository'] = repository

        # Write the adjusted package file back to the temporary directory and publish it.
        json_dump(package, package_file)

        # Add a boilerplate .gitignore file if the package does not already have one.
        path_to_gitignore = os.path.join(self._repo, '.gitignore')
        if not os.path.exists(path_to_gitignore):
            with open(path_to_gitignore, 'w') as f:
                f.write(DEFAULT_GITIGNORE)

        # Prefix the README.md with information about the proper repository.
        path_to_readme = os.path.join(self._repo, 'README.md')
        if os.path.exists(path_to_readme):
            with open(path_to_readme, 'r') as f:
                readme_contents = README_PREFIX + f.read()
        else:
            readme_contents = README_PREFIX
        with open(path_to_readme, 'w') as f:
            f.write(readme_contents)

        # Write out the packages to install for the nuclide-installer package.
        if self.get_package_name() == 'nuclide-installer':
            from publishers.nuclide_installer_config import generate_config
            installer_config_json = generate_config(
                new_semver, self._config.apm_package_names)
            with open(os.path.join(self._repo, 'lib', 'config.json'),
                      'w') as f:
                f.write(installer_config_json)

        # Now that all of the local changes have been written, commit them.
        tag_name = 'v' + new_semver
        self._git.commit_all(
            self._repo,
            'Committing changes in preparation for publishing %s' % tag_name)
        self._git.add_tag(self._repo, tag_name, 'Atom package %s.' % tag_name)
        # We commit the tag first because we should fail if the tag has already been pushed.
        self._git.push_tag(self._repo, tag_name)
        # Pushing to master is not strictly necessary, but it makes it easier to audit the changes
        # that have been made between versions over time.
        self._git.push_to_master(self._repo)
Ejemplo n.º 44
0
def load_dependencies():
    return json_load(get_dependencies_filename())
Ejemplo n.º 45
0
 def version(self):
     return json_load(os.path.join(self._root, 'package.json'))['version']
Ejemplo n.º 46
0
    def prepublish(self, new_version, atom_semver):
        self._repo = self._checkout_apm_repo()

        logging.info('Publishing %s to apm at version %s', self.get_package_name(), new_version)

        # Clean repo out, leaving .git metadata in place.
        logging.info('Cleaning repo for package %s', self.get_package_name())
        self.clean_repo()

        # Copy files from package into repo.
        package = self._config.package_directory
        logging.info('Copying package %s', self.get_package_name())
        self.copy_into_repo(package)

        # Load package.json and rewrite version number within it.
        # TODO (jpearce): reconcile with very similar npm code
        nil_semver = '0.0.0'
        new_semver = '0.0.%d' % new_version
        package_file = os.path.join(self._repo, 'package.json')
        package = json_load(package_file)
        if package['version'] != nil_semver:
            raise AssertionError('Local package %s was not at version 0' %
                                 self.get_package_name())
        package['version'] = new_semver

        # Update the versions of our local dependencies accordingly.
        for dependency_key in DEPENDENCIES_KEYS:
            if not dependency_key in package:
                continue
            for (dependency, version) in package[dependency_key].items():
                if not self._config.is_nuclide_npm_package(dependency):
                    continue
                if version != nil_semver:
                    raise AssertionError('Local dependency %s in package %s was not at version 0' %
                                         dependency, self.get_package_name())
                package[dependency_key][dependency] = new_semver

        # Update the version of the Atom engine required.
        package['engines'] = {'atom': '>=%s' % atom_semver}

        # Rewrite the repository URL.
        repository = 'https://github.com/facebooknuclideapm/%s' % self.get_package_name()
        package['repository'] = repository

        # Write the adjusted package file back to the temporary directory and publish it.
        json_dump(package, package_file)

        # Add a boilerplate .gitignore file if the package does not already have one.
        path_to_gitignore = os.path.join(self._repo, '.gitignore')
        if not os.path.exists(path_to_gitignore):
            with open(path_to_gitignore, 'w') as f:
                f.write(DEFAULT_GITIGNORE)

        # Prefix the README.md with information about the proper repository.
        path_to_readme = os.path.join(self._repo, 'README.md')
        if os.path.exists(path_to_readme):
            with open(path_to_readme, 'r') as f:
                readme_contents = README_PREFIX + f.read()
        else:
            readme_contents = README_PREFIX
        with open(path_to_readme, 'w') as f:
            f.write(readme_contents)

        # Write out the packages to install for the nuclide-installer package.
        if self.get_package_name() == 'nuclide-installer':
            from publishers.nuclide_installer_config import generate_config
            installer_config_json = generate_config(new_semver, self._config.apm_package_names)
            with open(os.path.join(self._repo, 'lib', 'config.json'), 'w') as f:
                f.write(installer_config_json)

        # Now that all of the local changes have been written, commit them.
        tag_name = 'v' + new_semver
        self._git.commit_all(self._repo,
                             'Committing changes in preparation for publishing %s' % tag_name)
        self._git.add_tag(self._repo, tag_name, 'Atom package %s.' % tag_name)
        # We commit the tag first because we should fail if the tag has already been pushed.
        self._git.push_tag(self._repo, tag_name)
        # Pushing to master is not strictly necessary, but it makes it easier to audit the changes
        # that have been made between versions over time.
        self._git.push_to_master(self._repo)
Ejemplo n.º 47
0
    def prepublish(self, new_version, atom_semver):
        self._repo = self._checkout_apm_repo()

        logging.info('Publishing %s to apm at version %s', self.get_package_name(), new_version)

        # Clean repo out, leaving .git metadata in place.
        logging.info('Cleaning repo for package %s', self.get_package_name())
        self.clean_repo()

        # Copy files from package into repo.
        package = self._config.package_directory
        logging.info('Copying package %s', self.get_package_name())
        self.copy_into_repo(package)

        # Make sure that standard boilerplate files are included in the repo.
        for name, src in self._boilerplate_files.items():
            shutil.copyfile(
                src,
                os.path.join(self._repo, name))

        # Load package.json and rewrite version number within it.
        package_file = os.path.join(self._repo, 'package.json')
        package = json_load(package_file)
        package = update_package_json_versions(self.get_package_name(), package,
            self._config.nuclide_npm_package_names, new_version)

        # Specify the license if it is not already specified.
        if 'license' not in package:
            package['license'] = 'SEE LICENSE IN LICENSE'

        # Update the version of the Atom engine required.
        package['engines'] = {'atom': '>=%s' % atom_semver}

        # Rewrite the repository URL.
        repository = 'https://github.com/facebooknuclideapm/%s' % self.get_package_name()
        package['repository'] = repository

        # Write the adjusted package file back to the temporary directory and publish it.
        json_dump(package, package_file)

        rewrite_shrinkwrap_file(self._repo,
            package, self._config.nuclide_npm_package_names, new_version)

        # Add a boilerplate .gitignore file if the package does not already have one.
        path_to_gitignore = os.path.join(self._repo, '.gitignore')
        if not os.path.exists(path_to_gitignore):
            with open(path_to_gitignore, 'w') as f:
                f.write(DEFAULT_GITIGNORE)

        # Prefix the README.md with information about the proper repository.
        path_to_readme = os.path.join(self._repo, 'README.md')
        if os.path.exists(path_to_readme):
            with open(path_to_readme, 'r') as f:
                readme_contents = README_PREFIX + f.read()
        else:
            readme_contents = README_PREFIX
        with open(path_to_readme, 'w') as f:
            f.write(readme_contents)

        # Write out the packages to install for the nuclide-installer package.
        if self.get_package_name() == 'nuclide-installer':
            from publishers.nuclide_installer_config import generate_config
            installer_config_json = generate_config(package['version'], self._config.apm_package_names)
            with open(os.path.join(self._repo, 'lib', 'config.json'), 'w') as f:
                f.write(installer_config_json)

        # Pre-transpile Babel files, as appropriate.
        self._transpiler.transpile_in_place(self.get_package_name(), self._repo)
Ejemplo n.º 48
0
 def version(self):
     return json_load(os.path.join(self._root, 'package.json'))['version']
Ejemplo n.º 49
0
def load_dependencies():
    return json_load(get_dependencies_filename())
Ejemplo n.º 50
0
    def create_package_publisher(path_to_nuclide_repo, master_tmpdir, github_access_token, npm_only):
        def get_list_of_packages(package_type):
            stdout = fs.cross_platform_check_output(
                    ['./scripts/dev/packages', '--package-type', package_type],
                    cwd=path_to_nuclide_repo)
            # Split by newlines and then remove the last element, which will be the empty string.
            return stdout.split('\n')[:-1]
        # These are lists of absolute paths to package.json files.
        node_packages = get_list_of_packages('Node')
        atom_packages = get_list_of_packages('Atom')

        # Ensure that nuclide-installer is listed last in atom_packages. We do not want to publish a
        # new version of the installer until we are sure that all of the packages it plans to
        # install have been published.
        for index, package_json_path in enumerate(atom_packages):
            package_name = json_load(package_json_path)['name']
            if package_name == 'nuclide-installer':
                del atom_packages[index]
                atom_packages.append(package_json_path)
                break

        # These are sets of package names.
        nuclide_npm_packages = set()
        nuclide_apm_packages = set()

        publishers = []
        git = Git()
        apm = Apm(git)
        npm = Npm()
        boilerplate_files = {
            'LICENSE': os.path.join(path_to_nuclide_repo, 'LICENSE'),
        }

        # Make sure that everything needed to run the transpile script is installed.
        subprocess.check_call(['npm', 'install'],
                              cwd=os.path.join(path_to_nuclide_repo, 'pkg/nuclide/node-transpiler'))
        transpile_script = os.path.join(path_to_nuclide_repo,
                                        'pkg/nuclide/node-transpiler/bin/transpile')
        transpiler = Transpiler.create_transpiler(path_to_nuclide_repo, transpile_script)

        def process_packages(packages, is_npm):
            for package_json in packages:
                package_name = json_load(package_json)['name']
                if is_npm:
                    nuclide_npm_packages.add(package_name)
                else:
                    nuclide_apm_packages.add(package_name)

                config = AbstractPublisherConfig(
                    package_name,
                    os.path.dirname(package_json),
                    nuclide_npm_packages,
                    nuclide_apm_packages)
                if is_npm:
                    publisher = NpmPublisher(config, npm, master_tmpdir, transpiler, boilerplate_files)
                else:
                    publisher = ApmPublisher(config, apm, master_tmpdir, transpiler, boilerplate_files, git, github_access_token)
                publishers.append(publisher)

        # Note that the resulting publishers array will be organized such that all
        # Node packages appear in topologically sorted order followed by all Atom packages.
        process_packages(node_packages, is_npm=True)
        if not npm_only:
            process_packages(atom_packages, is_npm=False)
        return PackagePublisher(publishers)
Ejemplo n.º 51
0
def create_config_for_package(path):
    '''Reads the package.json at `path` and returns a config created by
    `create_config_for_manifest`.
    '''
    return create_config_for_manifest(path, json_load(path))
Ejemplo n.º 52
0
def create_config_for_package(path):
    '''Reads the package.json at `path` and returns a config created by
    `create_config_for_manifest`.
    '''
    return create_config_for_manifest(path, json_load(path))
Ejemplo n.º 53
0
    def create_package_publisher(path_to_nuclide_repo, master_tmpdir,
                                 github_access_token):
        def get_list_of_packages(package_type):
            stdout = fs.cross_platform_check_output(
                ['./scripts/dev/packages', '--package-type', package_type],
                cwd=path_to_nuclide_repo)
            # Split by newlines and then remove the last element, which will be the empty string.
            return stdout.split('\n')[:-1]

        # These are lists of absolute paths to package.json files.
        node_packages = get_list_of_packages('Node')
        atom_packages = get_list_of_packages('Atom')

        # Ensure that nuclide-installer is listed last in atom_packages. We do not want to publish a
        # new version of the installer until we are sure that all of the packages it plans to
        # install have been published.
        for index, package_json_path in enumerate(atom_packages):
            package_name = json_load(package_json_path)['name']
            if package_name == 'nuclide-installer':
                del atom_packages[index]
                atom_packages.append(package_json_path)
                break

        # These are sets of package names.
        nuclide_npm_packages = set()
        nuclide_apm_packages = set()

        publishers = []
        git = Git()
        apm = Apm(git)
        npm = Npm()
        boilerplate_files = {
            'LICENSE': os.path.join(path_to_nuclide_repo, 'LICENSE'),
        }

        # Make sure that everything needed to run the transpile script is installed.
        subprocess.check_call(['npm', 'install'],
                              cwd=os.path.join(path_to_nuclide_repo,
                                               'pkg/nuclide/node-transpiler'))
        transpile_script = os.path.join(
            path_to_nuclide_repo, 'pkg/nuclide/node-transpiler/bin/transpile')
        transpiler = Transpiler.create_transpiler(path_to_nuclide_repo,
                                                  transpile_script)

        def process_packages(packages, is_npm):
            for package_json in packages:
                package_name = json_load(package_json)['name']
                if is_npm:
                    nuclide_npm_packages.add(package_name)
                else:
                    nuclide_apm_packages.add(package_name)

                config = AbstractPublisherConfig(package_name,
                                                 os.path.dirname(package_json),
                                                 nuclide_npm_packages,
                                                 nuclide_apm_packages)
                if is_npm:
                    publisher = NpmPublisher(config, npm, master_tmpdir,
                                             transpiler, boilerplate_files)
                else:
                    publisher = ApmPublisher(config, apm, master_tmpdir,
                                             transpiler, boilerplate_files,
                                             git, github_access_token)
                publishers.append(publisher)

        # Note that the resulting publishers array will be organized such that all
        # Node packages appear in topologically sorted order followed by all Atom packages.
        process_packages(node_packages, is_npm=True)
        process_packages(atom_packages, is_npm=False)
        return PackagePublisher(publishers)