Ejemplo n.º 1
0
          def publish(ivyxml_path):
            ivysettings = self.generate_ivysettings(published, publish_local=path)
            args = [
              '-settings', ivysettings,
              '-ivy', ivyxml_path,
              '-deliverto', '%s/[organisation]/[module]/ivy-[revision].xml' % self.outdir,
              '-publish', resolver,
              '-publishpattern', '%s/[organisation]/[module]/'
                                 '[artifact]-[revision](-[classifier]).[ext]' % self.outdir,
              '-revision', newver.version(),
              '-m2compatible',
            ]

            if LogOptions.stderr_log_level() == logging.DEBUG:
              args.append('-verbose')

            if self.snapshot:
              args.append('-overwrite')

            try:
              ivy = Bootstrapper.default_ivy()
              ivy.execute(jvm_options=jvm_args, args=args,
                          workunit_factory=self.context.new_workunit, workunit_name = 'jar-publish')
            except (Bootstrapper.Error, Ivy.Error) as e:
              raise TaskError('Failed to push %s! %s' % (jar_coordinate(jar, newver.version()), e))
Ejemplo n.º 2
0
          def publish(ivyxml_path):
            ivysettings = self.generate_ivysettings(published, publish_local=path)
            args = [
              '-settings', ivysettings,
              '-ivy', ivyxml_path,
              '-deliverto', '%s/[organisation]/[module]/ivy-[revision].xml' % self.outdir,
              '-publish', resolver,
              '-publishpattern', '%s/[organisation]/[module]/'
                                 '[artifact]-[revision](-[classifier]).[ext]' % self.outdir,
              '-revision', newver.version(),
              '-m2compatible',
            ]

            if LogOptions.stderr_log_level() == logging.DEBUG:
              args.append('-verbose')

            if self.snapshot:
              args.append('-overwrite')

            try:
              ivy = Bootstrapper.default_ivy()
              ivy.execute(jvm_options=jvm_args, args=args,
                          workunit_factory=self.context.new_workunit, workunit_name='jar-publish')
            except (Bootstrapper.Error, Ivy.Error) as e:
              raise TaskError('Failed to push %s! %s' % (jar_coordinate(jar, newver.version()), e))
Ejemplo n.º 3
0
    def exec_ivy(self,
                 target_workdir,
                 targets,
                 args,
                 confs=None,
                 ivy=None,
                 workunit_name='ivy',
                 workunit_factory=None,
                 symlink_ivyxml=False):

        ivy = ivy or Bootstrapper.default_ivy()
        if not isinstance(ivy, Ivy):
            raise ValueError(
                'The ivy argument supplied must be an Ivy instance, given %s of type %s'
                % (ivy, type(ivy)))

        ivyxml = os.path.join(target_workdir, 'ivy.xml')
        jars, excludes = self._calculate_classpath(targets)

        ivy_args = ['-ivy', ivyxml]

        confs_to_resolve = confs or ['default']
        ivy_args.append('-confs')
        ivy_args.extend(confs_to_resolve)

        ivy_args.extend(args)
        if not self._transitive:
            ivy_args.append('-notransitive')
        ivy_args.extend(self._args)

        def safe_link(src, dest):
            if os.path.exists(dest):
                os.unlink(dest)
            os.symlink(src, dest)

        with IvyUtils.ivy_lock:
            self._generate_ivy(targets, jars, excludes, ivyxml,
                               confs_to_resolve)
            runner = ivy.runner(jvm_options=self._jvm_options, args=ivy_args)
            try:
                result = util.execute_runner(runner,
                                             workunit_factory=workunit_factory,
                                             workunit_name=workunit_name)

                # Symlink to the current ivy.xml file (useful for IDEs that read it).
                if symlink_ivyxml:
                    ivyxml_symlink = os.path.join(self._work_dir, 'ivy.xml')
                    safe_link(ivyxml, ivyxml_symlink)

                if result != 0:
                    raise TaskError('Ivy returned %d' % result)
            except runner.executor.Error as e:
                raise TaskError(e)
Ejemplo n.º 4
0
  def exec_ivy(self,
               target_workdir,
               targets,
               args,
               confs=None,
               ivy=None,
               workunit_name='ivy',
               workunit_factory=None,
               symlink_ivyxml=False):

    ivy = ivy or Bootstrapper.default_ivy()
    if not isinstance(ivy, Ivy):
      raise ValueError('The ivy argument supplied must be an Ivy instance, given %s of type %s'
                       % (ivy, type(ivy)))

    ivyxml = os.path.join(target_workdir, 'ivy.xml')
    jars, excludes = self._calculate_classpath(targets)

    ivy_args = ['-ivy', ivyxml]

    confs_to_resolve = confs or ['default']
    ivy_args.append('-confs')
    ivy_args.extend(confs_to_resolve)

    ivy_args.extend(args)
    if not self._transitive:
      ivy_args.append('-notransitive')
    ivy_args.extend(self._args)

    def safe_link(src, dest):
      if os.path.exists(dest):
        os.unlink(dest)
      os.symlink(src, dest)

    with IvyUtils.ivy_lock:
      self._generate_ivy(targets, jars, excludes, ivyxml, confs_to_resolve)
      runner = ivy.runner(jvm_options=self._jvm_options, args=ivy_args)
      try:
        result = util.execute_runner(runner,
                                     workunit_factory=workunit_factory,
                                     workunit_name=workunit_name)

        # Symlink to the current ivy.xml file (useful for IDEs that read it).
        if symlink_ivyxml:
          ivyxml_symlink = os.path.join(self._work_dir, 'ivy.xml')
          safe_link(ivyxml, ivyxml_symlink)

        if result != 0:
          raise TaskError('Ivy returned %d' % result)
      except runner.executor.Error as e:
        raise TaskError(e)
Ejemplo n.º 5
0
  def run_antlrs(self, output_dir):
    args = [
      '-dependency', 'org.antlr', 'antlr', self.target.antlr_version,
      '-types', 'jar',
      '-main', 'org.antlr.Tool',
      '--', '-fo', output_dir
    ]
    for source in self.target.sources:
      abs_path = os.path.abspath(os.path.join(self.root, self.target.target_base, source))
      args.append(abs_path)

    try:
      ivy = Bootstrapper.default_ivy()
      ivy.execute(args=args)  # TODO: Needs a workunit, when we have a context here.
      return True
    except (Bootstrapper.Error, Ivy.Error) as e:
      print('ANTLR generation failed! %s' % e, file=sys.stderr)
      return False
Ejemplo n.º 6
0
    def run_antlrs(self, output_dir):
        args = [
            '-dependency', 'org.antlr', 'antlr', self.target.antlr_version,
            '-types', 'jar', '-main', 'org.antlr.Tool', '--', '-fo', output_dir
        ]
        for source in self.target.sources:
            abs_path = os.path.abspath(
                os.path.join(self.root, self.target.target_base, source))
            args.append(abs_path)

        try:
            ivy = Bootstrapper.default_ivy()
            ivy.execute(
                args=args
            )  # TODO: Needs a workunit, when we have a context here.
            return True
        except (Bootstrapper.Error, Ivy.Error) as e:
            print('ANTLR generation failed! %s' % e, file=sys.stderr)
            return False
Ejemplo n.º 7
0
    def mapjars(self, genmap, target, executor, workunit_factory=None):
        """
    Parameters:
      genmap: the jar_dependencies ProductMapping entry for the required products.
      target: the target whose jar dependencies are being retrieved.
    """
        mapdir = os.path.join(self.mapto_dir(), target.id)
        safe_mkdir(mapdir, clean=True)
        ivyargs = [
            '-retrieve',
            '%s/[organisation]/[artifact]/[conf]/'
            '[organisation]-[artifact]-[revision](-[classifier]).[ext]' %
            mapdir,
            '-symlink',
        ]
        self.exec_ivy(mapdir, [target],
                      ivyargs,
                      confs=target.configurations,
                      ivy=Bootstrapper.default_ivy(executor),
                      workunit_factory=workunit_factory,
                      workunit_name='map-jars')

        for org in os.listdir(mapdir):
            orgdir = os.path.join(mapdir, org)
            if os.path.isdir(orgdir):
                for name in os.listdir(orgdir):
                    artifactdir = os.path.join(orgdir, name)
                    if os.path.isdir(artifactdir):
                        for conf in os.listdir(artifactdir):
                            confdir = os.path.join(artifactdir, conf)
                            for f in os.listdir(confdir):
                                if self.is_mappable_artifact(org, name, f):
                                    # TODO(John Sirois): kill the org and (org, name) exclude mappings in favor of a
                                    # conf whitelist
                                    genmap.add(org, confdir).append(f)
                                    genmap.add((org, name), confdir).append(f)

                                    genmap.add(target, confdir).append(f)
                                    genmap.add((target, conf),
                                               confdir).append(f)
                                    genmap.add((org, name, conf),
                                               confdir).append(f)
Ejemplo n.º 8
0
  def mapjars(self, genmap, target, executor, workunit_factory=None):
    """
    Parameters:
      genmap: the jar_dependencies ProductMapping entry for the required products.
      target: the target whose jar dependencies are being retrieved.
    """
    mapdir = os.path.join(self.mapto_dir(), target.id)
    safe_mkdir(mapdir, clean=True)
    ivyargs = [
      '-retrieve', '%s/[organisation]/[artifact]/[conf]/'
                   '[organisation]-[artifact]-[revision](-[classifier]).[ext]' % mapdir,
      '-symlink',
    ]
    self.exec_ivy(mapdir,
                  [target],
                  ivyargs,
                  confs=target.configurations,
                  ivy=Bootstrapper.default_ivy(executor),
                  workunit_factory=workunit_factory,
                  workunit_name='map-jars')

    for org in os.listdir(mapdir):
      orgdir = os.path.join(mapdir, org)
      if os.path.isdir(orgdir):
        for name in os.listdir(orgdir):
          artifactdir = os.path.join(orgdir, name)
          if os.path.isdir(artifactdir):
            for conf in os.listdir(artifactdir):
              confdir = os.path.join(artifactdir, conf)
              for f in os.listdir(confdir):
                if self.is_mappable_artifact(org, name, f):
                  # TODO(John Sirois): kill the org and (org, name) exclude mappings in favor of a
                  # conf whitelist
                  genmap.add(org, confdir).append(f)
                  genmap.add((org, name), confdir).append(f)

                  genmap.add(target, confdir).append(f)
                  genmap.add((target, conf), confdir).append(f)
                  genmap.add((org, name, conf), confdir).append(f)
Ejemplo n.º 9
0
  def ivy_resolve(self, targets, executor=None, symlink_ivyxml=False, silent=False,
                  workunit_name=None, workunit_labels=None):

    if executor and not isinstance(executor, Executor):
      raise ValueError('The executor must be an Executor instance, given %s of type %s'
                       % (executor, type(executor)))
    ivy = Bootstrapper.default_ivy(java_executor=executor,
                                   bootstrap_workunit_factory=self.context.new_workunit)

    targets = set(targets)

    if not targets:
      return []

    work_dir = self.context.config.get('ivy-resolve', 'workdir')
    ivy_utils = IvyUtils(config=self.context.config,
                         options=self.context.options,
                         log=self.context.log)

    with self.invalidated(targets,
                          only_buildfiles=True,
                          invalidate_dependents=True,
                          silent=silent) as invalidation_check:
      global_vts = VersionedTargetSet.from_versioned_targets(invalidation_check.all_vts)
      target_workdir = os.path.join(work_dir, global_vts.cache_key.hash)
      target_classpath_file = os.path.join(target_workdir, 'classpath')
      raw_target_classpath_file = target_classpath_file + '.raw'
      raw_target_classpath_file_tmp = raw_target_classpath_file + '.tmp'
      # A common dir for symlinks into the ivy2 cache. This ensures that paths to jars
      # in artifact-cached analysis files are consistent across systems.
      # Note that we have one global, well-known symlink dir, again so that paths are
      # consistent across builds.
      symlink_dir = os.path.join(work_dir, 'jars')

      # Note that it's possible for all targets to be valid but for no classpath file to exist at
      # target_classpath_file, e.g., if we previously built a superset of targets.
      if invalidation_check.invalid_vts or not os.path.exists(raw_target_classpath_file):
        args = ['-cachepath', raw_target_classpath_file_tmp]

        def exec_ivy():
          ivy_utils.exec_ivy(
              target_workdir=target_workdir,
              targets=targets,
              args=args,
              ivy=ivy,
              workunit_name='ivy',
              workunit_factory=self.context.new_workunit,
              symlink_ivyxml=symlink_ivyxml)

        if workunit_name:
          with self.context.new_workunit(name=workunit_name, labels=workunit_labels or []):
            exec_ivy()
        else:
          exec_ivy()

        if not os.path.exists(raw_target_classpath_file_tmp):
          raise TaskError('Ivy failed to create classpath file at %s'
                          % raw_target_classpath_file_tmp)
        shutil.move(raw_target_classpath_file_tmp, raw_target_classpath_file)

        if self.artifact_cache_writes_enabled():
          self.update_artifact_cache([(global_vts, [raw_target_classpath_file])])

    # Make our actual classpath be symlinks, so that the paths are uniform across systems.
    # Note that we must do this even if we read the raw_target_classpath_file from the artifact
    # cache. If we cache the target_classpath_file we won't know how to create the symlinks.
    symlink_map = IvyUtils.symlink_cachepath(self.context.ivy_home, raw_target_classpath_file,
                                             symlink_dir, target_classpath_file)
    with Task.symlink_map_lock:
      all_symlinks_map = self.context.products.get_data('symlink_map') or defaultdict(list)
      for path, symlink in symlink_map.items():
        all_symlinks_map[os.path.realpath(path)].append(symlink)
      self.context.products.safe_create_data('symlink_map', lambda: all_symlinks_map)

    with IvyUtils.cachepath(target_classpath_file) as classpath:
      stripped_classpath = [path.strip() for path in classpath]
      return [path for path in stripped_classpath if ivy_utils.is_classpath_artifact(path)]
Ejemplo n.º 10
0
    def ivy_resolve(self,
                    targets,
                    executor=None,
                    symlink_ivyxml=False,
                    silent=False,
                    workunit_name=None,
                    workunit_labels=None):

        if executor and not isinstance(executor, Executor):
            raise ValueError(
                'The executor must be an Executor instance, given %s of type %s'
                % (executor, type(executor)))
        ivy = Bootstrapper.default_ivy(
            java_executor=executor,
            bootstrap_workunit_factory=self.context.new_workunit)

        targets = set(targets)

        if not targets:
            return []

        work_dir = self.context.config.get('ivy-resolve', 'workdir')
        ivy_utils = IvyUtils(config=self.context.config,
                             options=self.context.options,
                             log=self.context.log)

        with self.invalidated(targets,
                              only_buildfiles=True,
                              invalidate_dependents=True,
                              silent=silent) as invalidation_check:
            global_vts = VersionedTargetSet.from_versioned_targets(
                invalidation_check.all_vts)
            target_workdir = os.path.join(work_dir, global_vts.cache_key.hash)
            target_classpath_file = os.path.join(target_workdir, 'classpath')
            raw_target_classpath_file = target_classpath_file + '.raw'
            raw_target_classpath_file_tmp = raw_target_classpath_file + '.tmp'
            # A common dir for symlinks into the ivy2 cache. This ensures that paths to jars
            # in artifact-cached analysis files are consistent across systems.
            # Note that we have one global, well-known symlink dir, again so that paths are
            # consistent across builds.
            symlink_dir = os.path.join(work_dir, 'jars')

            # Note that it's possible for all targets to be valid but for no classpath file to exist at
            # target_classpath_file, e.g., if we previously built a superset of targets.
            if invalidation_check.invalid_vts or not os.path.exists(
                    raw_target_classpath_file):
                args = ['-cachepath', raw_target_classpath_file_tmp]

                def exec_ivy():
                    ivy_utils.exec_ivy(
                        target_workdir=target_workdir,
                        targets=targets,
                        args=args,
                        ivy=ivy,
                        workunit_name='ivy',
                        workunit_factory=self.context.new_workunit,
                        symlink_ivyxml=symlink_ivyxml)

                if workunit_name:
                    with self.context.new_workunit(name=workunit_name,
                                                   labels=workunit_labels
                                                   or []):
                        exec_ivy()
                else:
                    exec_ivy()

                if not os.path.exists(raw_target_classpath_file_tmp):
                    raise TaskError(
                        'Ivy failed to create classpath file at %s' %
                        raw_target_classpath_file_tmp)
                shutil.move(raw_target_classpath_file_tmp,
                            raw_target_classpath_file)

                if self.artifact_cache_writes_enabled():
                    self.update_artifact_cache([(global_vts,
                                                 [raw_target_classpath_file])])

        # Make our actual classpath be symlinks, so that the paths are uniform across systems.
        # Note that we must do this even if we read the raw_target_classpath_file from the artifact
        # cache. If we cache the target_classpath_file we won't know how to create the symlinks.
        symlink_map = IvyUtils.symlink_cachepath(self.context.ivy_home,
                                                 raw_target_classpath_file,
                                                 symlink_dir,
                                                 target_classpath_file)
        with Task.symlink_map_lock:
            all_symlinks_map = self.context.products.get_data(
                'symlink_map') or defaultdict(list)
            for path, symlink in symlink_map.items():
                all_symlinks_map[os.path.realpath(path)].append(symlink)
            self.context.products.safe_create_data('symlink_map',
                                                   lambda: all_symlinks_map)

        with IvyUtils.cachepath(target_classpath_file) as classpath:
            stripped_classpath = [path.strip() for path in classpath]
            return [
                path for path in stripped_classpath
                if ivy_utils.is_classpath_artifact(path)
            ]