Example #1
0
 def test_tar(self):
     self.round_trip(archiver('tar'), expected_ext='tar', empty_dirs=True)
     self.round_trip(archiver('tgz'),
                     expected_ext='tar.gz',
                     empty_dirs=True)
     self.round_trip(archiver('tbz2'),
                     expected_ext='tar.bz2',
                     empty_dirs=True)
Example #2
0
 def _extract_jar(self, jar_path):
   """Extracts the jar to a subfolder of workdir/extracted and returns the path to it."""
   with open(jar_path, 'rb') as f:
     outdir = os.path.join(self.workdir, 'extracted', sha1(f.read()).hexdigest())
   if not os.path.exists(outdir):
     _, extension = os.path.splitext(jar_path)
     archiver(extension[1:]).extract(jar_path, outdir)
     self.context.log.debug('Extracting archive at {jar_path}.'.format(jar_path=jar_path))
   else:
     self.context.log.debug('Archive already extracted at {jar_path}.'.format(jar_path=jar_path))
   return outdir
Example #3
0
  def test_zip_filter(self):
    def do_filter(path):
      return path == 'allowed.txt'

    with temporary_dir() as fromdir:
      touch(os.path.join(fromdir, 'allowed.txt'))
      touch(os.path.join(fromdir, 'disallowed.txt'))

      with temporary_dir() as archivedir:
        archive = archiver('zip').create(fromdir, archivedir, 'archive')
        with temporary_dir() as todir:
          archiver('zip').extract(archive, todir, filter_func=do_filter)
          self.assertEquals(set(['allowed.txt']), self._listtree(todir, empty_dirs=False))
Example #4
0
    def check_archive_with_flags(archive_format, dereference):
      with temporary_dir() as fromdir:
        filename = os.path.join(fromdir, 'a')
        linkname = os.path.join(fromdir, 'link_to_a')
        touch(filename)
        relative_symlink(filename, linkname)

        with temporary_dir() as archivedir:
          archive = archiver(archive_format).create(fromdir, archivedir, 'archive', dereference=dereference)
          with temporary_dir() as todir:
            archiver(archive_format).extract(archive, todir)
            extracted_linkname = os.path.join(todir, 'link_to_a')
            assertion = self.assertFalse if dereference else self.assertTrue
            assertion(os.path.islink(extracted_linkname))
            assertion(os.path.samefile(extracted_linkname, os.path.join(todir, 'a')))
Example #5
0
  def execute(self):
    bundleable_js = self.context.products.get_data('bundleable_js')
    bundle_archive_product = self.context.products.get('deployable_archives')
    dirutil.safe_mkdir(self._outdir)  # Make sure dist dir is present.

    for target in self.context.target_roots:
      if self.is_node_bundle(target):
        archiver = archive.archiver(target.payload.archive)
        for _, abs_paths in bundleable_js[target.node_module].abs_paths():
          for abs_path in abs_paths:
            # build_dir is a symlink.  Since dereference option for tar is set to False, we need to
            # dereference manually to archive the linked build dir.
            build_dir = os.path.realpath(abs_path)
            self.context.log.debug('archiving {}'.format(build_dir))
            archivepath = archiver.create(
              build_dir,
              self._outdir,
              target.package_name,
              prefix=None,
              dereference=False
            )
            bundle_archive_product.add(
              target, os.path.dirname(archivepath)).append(os.path.basename(archivepath))
            self.context.log.info(
              'created {}'.format(os.path.relpath(archivepath, get_buildroot())))
Example #6
0
  def execute(self):
    targets_to_bundle = self.context.targets(self.App.is_app)

    if self.get_options().use_basename_prefix:
      self.check_basename_conflicts([t for t in self.context.target_roots if t in targets_to_bundle])

    with self.invalidated(targets_to_bundle, invalidate_dependents=True) as invalidation_check:
      jvm_bundles_product = self.context.products.get('jvm_bundles')
      bundle_archive_product = self.context.products.get('deployable_archives')
      jvm_archive_product = self.context.products.get('jvm_archives')

      for vt in invalidation_check.all_vts:
        app = self.App.create_app(vt.target,
                                  self._resolved_option(vt.target, 'deployjar'),
                                  self._resolved_option(vt.target, 'archive'))
        archiver = archive.archiver(app.archive) if app.archive else None

        bundle_dir = self._get_bundle_dir(app, vt.results_dir)
        ext = archive.archive_extensions.get(app.archive, app.archive)
        filename = '{}.{}'.format(app.id, ext)
        archive_path = os.path.join(vt.results_dir, filename) if app.archive else ''
        if not vt.valid:
          self.bundle(app, vt.results_dir)
          if app.archive:
            archiver.create(bundle_dir, vt.results_dir, app.id)

        self._add_product(jvm_bundles_product, app, bundle_dir)
        if archiver:
          self._add_product(bundle_archive_product, app, archive_path)
          self._add_product(jvm_archive_product, app, archive_path)

        # For root targets, create symlink.
        if vt.target in self.context.target_roots:
          self._store_results(vt, bundle_dir, archive_path, app)
Example #7
0
    def execute(self):
        targets_to_bundle = self.context.targets(PythonApp.is_python_app)

        with self.invalidated(
                targets_to_bundle,
                invalidate_dependents=True) as invalidation_check:
            bundle_archive_product = self.context.products.get(
                self._DEPLOYABLE_ARCHIVES)

            for vt in invalidation_check.all_vts:
                bundle_dir = self.get_bundle_dir(vt.target.id, vt.results_dir)
                archive_format = self.resolved_option(self.get_options(),
                                                      vt.target, 'archive')
                archiver = archive.archiver(
                    archive_format) if archive_format else None
                archive_path = self._get_archive_path(vt, archive_format)

                if not vt.valid:  # Only recreate the bundle/archive if it's changed
                    self._bundle(vt.target, bundle_dir)
                    if archiver:
                        archiver.create(bundle_dir, vt.results_dir,
                                        vt.target.id)
                        self.context.log.info('created archive {}'.format(
                            os.path.relpath(archive_path, get_buildroot())))

                if archiver:
                    bundle_archive_product.add(
                        vt.target, os.path.dirname(archive_path)).append(
                            os.path.basename(archive_path))

                if vt.target in self.context.target_roots:  # Always publish bundle/archive in dist
                    self.publish_results(self.get_options().pants_distdir,
                                         False, vt, bundle_dir, archive_path,
                                         vt.target.id, archive_format)
Example #8
0
  def execute(self):
    targets_to_bundle = self.context.targets(self.App.is_app)

    if self.get_options().use_basename_prefix:
      self.check_basename_conflicts([t for t in self.context.target_roots if t in targets_to_bundle])

    with self.invalidated(targets_to_bundle, invalidate_dependents=True) as invalidation_check:
      jvm_bundles_product = self.context.products.get('jvm_bundles')
      bundle_archive_product = self.context.products.get('deployable_archives')
      jvm_archive_product = self.context.products.get('jvm_archives')

      for vt in invalidation_check.all_vts:
        app = self.App.create_app(vt.target,
                                  self._resolved_option(vt.target, 'deployjar'),
                                  self._resolved_option(vt.target, 'archive'))
        archiver = archive.archiver(app.archive) if app.archive else None

        bundle_dir = self._get_bundle_dir(app, vt.results_dir)
        ext = archive.archive_extensions.get(app.archive, app.archive)
        filename = '{}.{}'.format(app.id, ext)
        archive_path = os.path.join(vt.results_dir, filename) if app.archive else ''
        if not vt.valid:
          self.bundle(app, vt.results_dir)
          if app.archive:
            archiver.create(bundle_dir, vt.results_dir, app.id)

        self._add_product(jvm_bundles_product, app, bundle_dir)
        if archiver:
          self._add_product(bundle_archive_product, app, archive_path)
          self._add_product(jvm_archive_product, app, archive_path)

        # For root targets, create symlink.
        if vt.target in self.context.target_roots:
          self._store_results(vt, bundle_dir, archive_path, app)
Example #9
0
    def test_zip_filter(self):
        def do_filter(path):
            return path == 'allowed.txt'

        with temporary_dir() as fromdir:
            touch(os.path.join(fromdir, 'allowed.txt'))
            touch(os.path.join(fromdir, 'disallowed.txt'))

            with temporary_dir() as archivedir:
                archive = archiver('zip').create(fromdir, archivedir,
                                                 'archive')
                with temporary_dir() as todir:
                    archiver('zip').extract(archive,
                                            todir,
                                            filter_func=do_filter)
                    self.assertEquals(set(['allowed.txt']),
                                      self._listtree(todir, empty_dirs=False))
Example #10
0
 def select(self, supportdir, version, name, platform_dependent,
            archive_type):
     """Fetches a file, unpacking it if necessary."""
     if archive_type is None:
         return self._select_file(supportdir, version, name,
                                  platform_dependent)
     selected_archiver = archiver(archive_type)
     return self._select_archive(supportdir, version, name,
                                 platform_dependent, selected_archiver)
 def _extract_archive(self, archive_path):
   with temporary_dir() as temp_dir:
     _, extension = os.path.splitext(archive_path)
     print (extension)
     if extension == '.jar':
       extraction_archiver = archiver('zip')
     else:
       extraction_archiver = archiver_for_path(os.path.basename(archive_path))
     extraction_archiver.extract(archive_path, temp_dir)
     yield temp_dir
 def execute(self):
     archiver = archive.archiver(self._archiver_type) if self._archiver_type else None
     for target in self.context.target_roots:
         for app in map(self.App, filter(self.App.is_app, [target])):
             basedir = self.bundle(app)
             if archiver:
                 archivepath = archiver.create(
                     basedir, self._outdir, app.basename, prefix=app.basename if self._prefix else None
                 )
                 self.context.log.info("created {}".format(os.path.relpath(archivepath, get_buildroot())))
 def _extract_archive(self, archive_path):
     with temporary_dir() as temp_dir:
         _, extension = os.path.splitext(archive_path)
         print(extension)
         if extension == '.jar':
             extraction_archiver = archiver('zip')
         else:
             extraction_archiver = archiver_for_path(
                 os.path.basename(archive_path))
         extraction_archiver.extract(archive_path, temp_dir)
         yield temp_dir
Example #14
0
    def execute(self):
        use_basename_prefix = self.get_options().use_basename_prefix
        if use_basename_prefix:
            # NB(peiyu) This special casing is confusing especially given we already fail
            # when duplicate basenames are detected. It's added because of the existing
            # user experience. Turns out a `jvm_app` that depends on another `jvm_binary`
            # of the same basename is fairly common. In this case, using just
            # `target_roots` instead of all transitive targets will reduce the chance users
            # see their bundle command fail due to basename conflicts. We should eventually
            # get rid of this special case.
            targets_to_bundle = self.context.target_roots
        else:
            targets_to_bundle = self.context.targets()

        apps = []
        for target in targets_to_bundle:
            if self.App.is_app(target):
                apps.append(
                    self.App.create_app(
                        target,
                        self._resolved_option(target, 'deployjar'),
                        self._resolved_option(target, 'archive'),
                        use_basename_prefix=use_basename_prefix))

        if use_basename_prefix:
            self.check_basename_conflicts(apps)

        # NB(peiyu): performance hack to convert loose directories in classpath into jars. This is
        # more efficient than loading them as individual files.
        runtime_classpath = self.context.products.get_data('runtime_classpath')
        targets_to_consolidate = self.find_consolidate_classpath_candidates(
            runtime_classpath,
            self.context.targets(**self._target_closure_kwargs),
        )
        self.consolidate_classpath(targets_to_consolidate, runtime_classpath)

        for app in apps:
            archiver = archive.archiver(app.archive) if app.archive else None

            basedir = self.bundle(app)
            # NB(Eric Ayers): Note that this product is not housed/controlled under .pants.d/  Since
            # the bundle is re-created every time, this shouldn't cause a problem, but if we ever
            # expect the product to be cached, a user running an 'rm' on the dist/ directory could
            # cause inconsistencies.
            jvm_bundles_product = self.context.products.get('jvm_bundles')
            jvm_bundles_product.add(app.target,
                                    os.path.dirname(basedir)).append(
                                        os.path.basename(basedir))
            if archiver:
                archivepath = archiver.create(basedir,
                                              self.get_options().pants_distdir,
                                              app.basename)
                self.context.log.info('created {}'.format(
                    os.path.relpath(archivepath, get_buildroot())))
Example #15
0
        def check_archive_with_flags(archive_format, dereference):
            with temporary_dir() as fromdir:
                filename = os.path.join(fromdir, 'a')
                linkname = os.path.join(fromdir, 'link_to_a')
                touch(filename)
                relative_symlink(filename, linkname)

                with temporary_dir() as archivedir:
                    archive = archiver(archive_format).create(
                        fromdir,
                        archivedir,
                        'archive',
                        dereference=dereference)
                    with temporary_dir() as todir:
                        archiver(archive_format).extract(archive, todir)
                        extracted_linkname = os.path.join(todir, 'link_to_a')
                        assertion = self.assertFalse if dereference else self.assertTrue
                        assertion(os.path.islink(extracted_linkname))
                        assertion(
                            os.path.samefile(extracted_linkname,
                                             os.path.join(todir, 'a')))
Example #16
0
 def execute(self):
   archiver = archive.archiver(self._archiver_type) if self._archiver_type else None
   for target in self.context.target_roots:
     for app in map(self.App, filter(self.App.is_app, [target])):
       basedir = self.bundle(app)
       if archiver:
         archivepath = archiver.create(
           basedir,
           self._outdir,
           app.basename,
           prefix=app.basename if self._prefix else None
         )
         self.context.log.info('created %s' % os.path.relpath(archivepath, get_buildroot()))
Example #17
0
  def execute(self):
    use_basename_prefix = self.get_options().use_basename_prefix
    if use_basename_prefix:
      # NB(peiyu) This special casing is confusing especially given we already fail
      # when duplicate basenames are detected. It's added because of the existing
      # user experience. Turns out a `jvm_app` that depends on another `jvm_binary`
      # of the same basename is fairly common. In this case, using just
      # `target_roots` instead of all transitive targets will reduce the chance users
      # see their bundle command fail due to basename conflicts. We should eventually
      # get rid of this special case.
      targets_to_bundle = self.context.target_roots
    else:
      targets_to_bundle = self.context.targets()

    apps = []
    for target in targets_to_bundle:
      if self.App.is_app(target):
        apps.append(self.App.create_app(target,
                                        self._resolved_option(target, 'deployjar'),
                                        self._resolved_option(target, 'archive'),
                                        use_basename_prefix=use_basename_prefix))

    if use_basename_prefix:
      self.check_basename_conflicts(apps)

    # NB(peiyu): performance hack to convert loose directories in classpath into jars. This is
    # more efficient than loading them as individual files.
    runtime_classpath = self.context.products.get_data('runtime_classpath')
    targets_to_consolidate = self.find_consolidate_classpath_candidates(
      runtime_classpath,
      self.context.targets(**self._target_closure_kwargs),
    )
    self.consolidate_classpath(targets_to_consolidate, runtime_classpath)

    for app in apps:
      archiver = archive.archiver(app.archive) if app.archive else None

      basedir = self.bundle(app)
      # NB(Eric Ayers): Note that this product is not housed/controlled under .pants.d/  Since
      # the bundle is re-created every time, this shouldn't cause a problem, but if we ever
      # expect the product to be cached, a user running an 'rm' on the dist/ directory could
      # cause inconsistencies.
      jvm_bundles_product = self.context.products.get('jvm_bundles')
      jvm_bundles_product.add(app.target, os.path.dirname(basedir)).append(os.path.basename(basedir))
      if archiver:
        archivepath = archiver.create(
          basedir,
          self.get_options().pants_distdir,
          app.basename
        )
        self.context.log.info('created {}'.format(os.path.relpath(archivepath, get_buildroot())))
Example #18
0
 def execute(self, _):
   archiver = archive.archiver(self.archiver_type) if self.archiver_type else None
   for target in self.context.target_roots:
     for app in map(self.App, filter(self.App.is_app, target.resolve())):
       basedir = self.bundle(app)
       if archiver:
         archivemap = self.context.products.get(self.archiver_type)
         archivepath = archiver.create(
           basedir,
           self.outdir,
           app.basename,
           prefix=app.basename if self.prefix else None
         )
         archivemap.add(app, self.outdir, [archivepath])
         self.context.log.info('created %s' % os.path.relpath(archivepath, get_buildroot()))
Example #19
0
 def execute(self):
     archiver = archive.archiver(self._archiver_type) if self._archiver_type else None
     for target in self.context.target_roots:
         for app in map(self.App, filter(self.App.is_app, [target])):
             basedir = self.bundle(app)
             # NB(Eric Ayers): Note that this product is not housed/controlled under .pants.d/  Since
             # the bundle is re-created every time, this shouldn't cause a problem, but if we ever
             # expect the product to be cached, a user running an 'rm' on the dist/ directory could
             # cause inconsistencies.
             jvm_bundles_product = self.context.products.get("jvm_bundles")
             jvm_bundles_product.add(target, os.path.dirname(basedir)).append(os.path.basename(basedir))
             if archiver:
                 archivepath = archiver.create(
                     basedir, self._outdir, app.basename, prefix=app.basename if self._prefix else None
                 )
                 self.context.log.info("created {}".format(os.path.relpath(archivepath, get_buildroot())))
Example #20
0
  def execute(self):
    # NB(peiyu): performance hack to convert loose directories in classpath into jars. This is
    # more efficient than loading them as individual files.
    runtime_classpath = self.context.products.get_data('runtime_classpath')

    # TODO (from mateor) The consolidate classpath is something that we could do earlier in the
    # pipeline and it would be nice to just add those unpacked classed to a product and get the
    # consolidated classpath for free.
    targets_to_consolidate = self.find_consolidate_classpath_candidates(
      runtime_classpath,
      self.context.targets(**self._target_closure_kwargs),
    )
    self.consolidate_classpath(targets_to_consolidate, runtime_classpath)

    targets_to_bundle = self.context.targets(self.App.is_app)

    if self.get_options().use_basename_prefix:
      self.check_basename_conflicts([t for t in self.context.target_roots if t in targets_to_bundle])

    with self.invalidated(targets_to_bundle, invalidate_dependents=True) as invalidation_check:
      jvm_bundles_product = self.context.products.get('jvm_bundles')
      bundle_archive_product = self.context.products.get('deployable_archives')
      jvm_archive_product = self.context.products.get('jvm_archives')

      for vt in invalidation_check.all_vts:
        app = self.App.create_app(vt.target,
                                  self._resolved_option(vt.target, 'deployjar'),
                                  self._resolved_option(vt.target, 'archive'))
        archiver = archive.archiver(app.archive) if app.archive else None

        bundle_dir = self._get_bundle_dir(app, vt.results_dir)
        ext = archive.archive_extensions.get(app.archive, app.archive)
        filename = '{}.{}'.format(app.id, ext)
        archive_path = os.path.join(vt.results_dir, filename) if app.archive else ''
        if not vt.valid:
          self.bundle(app, vt.results_dir)
          if app.archive:
            archiver.create(bundle_dir, vt.results_dir, app.id)

        self._add_product(jvm_bundles_product, app, bundle_dir)
        if archiver:
          self._add_product(bundle_archive_product, app, archive_path)
          self._add_product(jvm_archive_product, app, archive_path)

        # For root targets, create symlink.
        if vt.target in self.context.target_roots:
          self._store_results(vt, bundle_dir, archive_path, app)
Example #21
0
 def execute(self):
   archiver = archive.archiver(self._archiver_type) if self._archiver_type else None
   for target in self.context.target_roots:
     for app in map(self.App, filter(self.App.is_app, [target])):
       basedir = self.bundle(app)
       # NB(Eric Ayers): Note that this product is not housed/controlled under .pants.d/  Since 
       # the bundle is re-created every time, this shouldn't cause a problem, but if we ever
       # expect the product to be cached, a user running an 'rm' on the dist/ directory could 
       # cause inconsistencies.
       jvm_bundles_product = self.context.products.get('jvm_bundles')
       jvm_bundles_product.add(target, os.path.dirname(basedir)).append(os.path.basename(basedir))
       if archiver:
         archivepath = archiver.create(
           basedir,
           self._outdir,
           app.basename,
           prefix=app.basename if self._prefix else None
         )
         self.context.log.info('created {}'.format(os.path.relpath(archivepath, get_buildroot())))
Example #22
0
 def test_zip(self):
   self.round_trip(archiver('zip'), expected_ext='zip', empty_dirs=False)
Example #23
0
 def test_tar(self):
   self.round_trip(archiver('tar'), expected_ext='tar', empty_dirs=True)
   self.round_trip(archiver('tgz'), expected_ext='tar.gz', empty_dirs=True)
   self.round_trip(archiver('tbz2'), expected_ext='tar.bz2', empty_dirs=True)
Example #24
0
 def test_zip(self):
     self.round_trip(archiver('zip'), expected_ext='zip', empty_dirs=False)