Example #1
0
    def copy_history(self, dest, job_ids=None):
        """Copy history logs from master.
        
        Args:
          dest (str):
            The path of the local dir where the logs will be copied.
          job_ids (list of str, optional):
            A list with the ids of the jobs for which the history should be
            copied. If nothing is passed, the history of all jobs is copied.
        """

        if not os.path.exists(dest):
            logger.warning("Destination directory " + dest +
                           " does not exist. It will be created")
            os.makedirs(dest)

        history_dir = os.path.join(self.logs_dir, "history")
        if job_ids:
            pattern = " -o ".join("-name " + jid + "*" for jid in job_ids)
            list_dirs = SshProcess("find " + history_dir + " " + pattern,
                                   self.master)
            list_dirs.run()
        else:
            list_dirs = SshProcess("find " + history_dir + " -name job_*",
                                   self.master)
            list_dirs.run()

        remote_files = []
        for line in list_dirs.stdout.splitlines():
            remote_files.append(line)

        action = Get([self.master], remote_files, dest)
        action.run()
Example #2
0
    def copy_history(self, dest, job_ids=None):
        """Copy history logs from master.
        
        Args:
          dest (str):
            The path of the local dir where the logs will be copied.
          job_ids (list of str, optional):
            A list with the ids of the jobs for which the history should be
            copied. If nothing is passed, the history of all jobs is copied.
        """

        if not os.path.exists(dest):
            logger.warning("Destination directory " + dest +
                           " does not exist. It will be created")
            os.makedirs(dest)

        history_dir = os.path.join(self.logs_dir, "history")
        if job_ids:
            pattern = " -o ".join("-name " + jid + "*" for jid in job_ids)
            list_dirs = SshProcess("find " + history_dir + " " + pattern,
                                   self.master)
            list_dirs.run()
        else:
            list_dirs = SshProcess("find " + history_dir + " -name job_*",
                                   self.master)
            list_dirs.run()

        remote_files = []
        for line in list_dirs.stdout.splitlines():
            remote_files.append(line)

        action = Get([self.master], remote_files, dest)
        action.run()
Example #3
0
def install_os(reconfigure, tags = None):
    update_config_state()

    # Clone or pull Kolla
    if os.path.isdir('kolla'):
        logger.info("Remove previous Kolla installation")
        kolla_path = os.path.join(SCRIPT_PATH, "kolla")
        call("rm -rf %s" % kolla_path, shell=True)

    logger.info("Cloning Kolla")
    call("cd %s ; git clone %s -b %s > /dev/null" % (SCRIPT_PATH, KOLLA_REPO, KOLLA_BRANCH), shell=True)

    logger.warning("Patching kolla, this should be \
            deprecated with the new version of Kolla")

    playbook = os.path.join(SCRIPT_PATH, "ansible/patches.yml")
    inventory_path = os.path.join(SYMLINK_NAME, 'multinode')
    run_ansible([playbook], inventory_path, STATE['config'])

    kolla_cmd = [os.path.join(SCRIPT_PATH, "kolla", "tools", "kolla-ansible")]

    if reconfigure:
        kolla_cmd.append('reconfigure')
    else:
        kolla_cmd.append('deploy')

    kolla_cmd.extend(["-i", "%s/multinode" % SYMLINK_NAME,
                  "--passwords", "%s/passwords.yml" % SYMLINK_NAME,
                  "--configdir", "%s" % SYMLINK_NAME])

    if tags is not None:
        kolla_cmd.extend(["--tags", args])

    call(kolla_cmd)
Example #4
0
    def workflow(self, comb):
        """ """
        comb_ok = False
        logger.info(slugify(comb) + \
                             ' starts to compile')
        try:
	    export = "source /opt/intel/bin/compilervars.sh intel64; "
	    
	    src_directory = "/home/arrouan/workspace/aevol/git/world/aevol/"
	    
	    bin_directory = "/home/arrouan/workspace/aevol/compiled_binary/"
    
	    configure_option = "--with-tracing --without-x"
	    
	    if comb['parallel'] == 'tbb':
	      configure_option += " --with-tbb"
	      
	    if comb['blas'] == 'openblas':
	      configure_option += " --with-blas"
	    elif comb['blas'] == 'mkl':
	      configure_option += " --with-mkl"
	    elif comb['blas'] == 'atlas':
	      configure_option += " --with-atlas"
	        
	    if comb['experiment'] == 'raevol':
	      configure_option += " --with-raevol"
	      
	    if comb['compilator'] == 'intel':
	      configure_option += " CXX=icc"
	      
	    full_bin_directory = bin_directory + comb['experiment']+'_'+comb['compilator']+'_'+comb['parallel']+'_'+comb['blas']
	    
	    try:
	      os.mkdir(full_bin_directory)
            except:
	      for f in os.listdir(full_bin_directory):
		os.remove(full_bin_directory + "/" + f)
		
	    p = Process(export+'cd '+src_directory+'; autoreconf; ./configure '+configure_option+'; make clean; make; cp src/aevol_run '+full_bin_directory+'/; cp src/aevol_create '+full_bin_directory+'/')
	    p.shell = True
	    #
	    p.run()
	    
	    print p.stdout
	    
            comb_ok = True
        finally:
            if comb_ok:
                self.sweeper.done(comb)
	        logger.info(slugify(comb) + \
                             ' has been done')
            else:
                self.sweeper.cancel(comb)
                logger.warning(slugify(comb) + \
                            ' has been canceled')
        logger.info(style.step('%s Remaining'),
                        len(self.sweeper.get_remaining()))
Example #5
0
    def workflow(self, comb):
        """
            Compute one application launch 
            using a given parameter group
        """
        comb_ok = False
        try:
            # Generate configuration file needed by MPI processes
            logger.info("Generating assembly file...")
            py = comb['cores'] / comb['px']
            prepare = Process('cd %s && python %s %d %d %d %d %d %s app.lad' % 
                (self.workingPath, self.genLadScript, comb['datasize'], comb['datasize'], 
                    comb['datasize'], comb['px'], py, comb['transposition']))
            prepare.shell = True
            prepare.run()

            # Generate the MPI host file
            mfile = self.generate_machine_file()

            # Start L2C
            lad = "./app.lad"
            logger.info("Computing...")
            res = Process("export OAR_JOB_KEY_FILE=~/.oar_key ; cd %s && l2c_loader -M,-machinefile,%s --mpi -c %d %s" % (self.workingPath, mfile, comb['cores'], lad))
            res.shell = True
            res.stdout_handlers.append(os.path.join(self.result_dir, slugify(comb) + '.out'))
            res.stdout_handlers.append(sys.stdout)
            res.stderr_handlers.append(os.path.join(self.result_dir, slugify(comb) + '.err'))
            res.stderr_handlers.append(sys.stderr)
            res.run()
            if not res.ok:
                logger.error('Bad L2C termination')
                raise Exception('Bad L2C termination')
            if len(res.stderr) > 0: # WARNING: when L2C cannot find the LAD file or something strange like this
                logger.warning('Not empty error output')

            # Clean configuration files
            logger.info("Removing assembly files...")
            res = Process('cd %s && rm -f app.lad*' % self.workingPath)
            res.shell = True
            res.run()
                
            comb_ok = True
        except Exception:
            pass
        finally:
            if comb_ok:
                self.sweeper.done(comb)
                logger.info(style.host(slugify(comb)) + ' has been done')
            else:
                self.sweeper.cancel(comb)
                logger.warning(style.host(slugify(comb)) + ' has been canceled')
        
            logger.info(style.step('%s Remaining'),
                        len(self.sweeper.get_remaining()))
Example #6
0
    def workflow(self, comb, host, comb_dir):
        """ """
        comb_ok = False
        thread_name = style.Thread(host.split('.')[0]) + ': '
        logger.info(thread_name + 'Starting combination ' + slugify(comb))

        try:
            logger.info(thread_name + 'Generate conf file')
            param_str = self.create_string(comb)

            Remote(
                "python /home/Work/sgcbntier/paasage_demo/xml_gen_execo.py --cb "
                + param_str, [host]).run()

            logger.info(thread_name + 'Run code')
            Remote(
                "cd /home/Work/sgcbntier/paasage_demo/ ; python run_all_execo.py --cb %s"
                % param_str, [host]).run()

            logger.info(thread_name + 'Get results')

            traceFile = "ntier_" + param_str
            get_results = Get([host], [
                "/home/Work/sgcbntier/paasage_demo/csv/REQTASK_" + traceFile +
                ".csv"
            ],
                              local_location=comb_dir).run()

            for p in get_results.processes:
                if not p.ok:
                    logger.error(
                        host +
                        ': Unable to retrieve the files for combination %s',
                        slugify(comb))
                    exit()

            comb_ok = True
        finally:
            if comb_ok:
                self.sweeper.done(comb)
                logger.info(thread_name + ': ' + slugify(comb) + \
                             ' has been done')
            else:
                self.sweeper.cancel(comb)
                logger.warning(thread_name + ': ' + slugify(comb) + \
                            ' has been canceled')
        logger.info(style.step('%s Remaining'),
                    len(self.sweeper.get_remaining()))
Example #7
0
    def copy_history(self, dest, job_ids=None):
        """Copy history logs from dfs.

        Args:
          dest (str):
            The path of the local dir where the logs will be copied.
          job_ids (list of str, optional):
            A list with the ids of the jobs for which the history should be
            copied. If nothing is passed, the history of all jobs is copied.
        """

        if not os.path.exists(dest):
            logger.warning("Destination directory " + dest +
                           " does not exist. It will be created")
            os.makedirs(dest)

        # Dirs used
        user_login = getpass.getuser()
        hist_dfs_dir = "/tmp/hadoop-yarn/staging/history/done_intermediate/" + \
                       user_login
        hist_tmp_dir = "/tmp/hadoop_hist"

        # Remove file in tmp dir if exists
        proc = SshProcess("rm -rf " + hist_tmp_dir, self.master)
        proc.run()

        # Get files in master
        if job_ids:
            proc = SshProcess("mkdir " + hist_tmp_dir, self.master)
            proc.run()
            for jid in job_ids:
                self.execute("fs -get " + hist_dfs_dir + "/" + jid + "* " +
                             hist_tmp_dir,
                             verbose=False)
        else:
            self.execute("fs -get " + hist_dfs_dir + " " + hist_tmp_dir,
                         verbose=False)

        # Copy files from master
        action = Get([self.master], [hist_tmp_dir], dest)
        action.run()
Example #8
0
    def copy_history(self, dest, job_ids=None):
        """Copy history logs from dfs.

        Args:
          dest (str):
            The path of the local dir where the logs will be copied.
          job_ids (list of str, optional):
            A list with the ids of the jobs for which the history should be
            copied. If nothing is passed, the history of all jobs is copied.
        """

        if not os.path.exists(dest):
            logger.warning("Destination directory " + dest +
                           " does not exist. It will be created")
            os.makedirs(dest)

        # Dirs used
        user_login = getpass.getuser()
        hist_dfs_dir = "/tmp/hadoop-yarn/staging/history/done_intermediate/" + \
                       user_login
        hist_tmp_dir = "/tmp/hadoop_hist"

        # Remove file in tmp dir if exists
        proc = SshProcess("rm -rf " + hist_tmp_dir, self.master)
        proc.run()

        # Get files in master
        if job_ids:
            proc = SshProcess("mkdir " + hist_tmp_dir, self.master)
            proc.run()
            for jid in job_ids:
                self.execute("fs -get " + hist_dfs_dir + "/" + jid + "* " +
                             hist_tmp_dir, verbose=False)
        else:
            self.execute("fs -get " + hist_dfs_dir + " " + hist_tmp_dir,
                         verbose=False)

        # Copy files from master
        action = Get([self.master], [hist_tmp_dir], dest)
        action.run()
    def run(self):
        """ """
        token = 'bRIJb9jp5igAAAAAAAAACc5QzQ619Vp0pYa2PdIrt0q2y0qFyJgwrKvtzuTp3Sz_'
        client = dropbox.client.DropboxClient(token)
        parameters = {'size': igeom(128, 2048, 5),
                      'db_if': ['rest', 'sdk']}
        combs = sweep(parameters)
        sweeper = ParamSweeper(self.result_dir + "/sweeps", combs)

        f = open(self.result_dir + '/results.txt', 'w')
        while len(sweeper.get_remaining()) > 0:
            comb = sweeper.get_next()

            logger.info('Treating combination %s', pformat(comb))
            comb_dir = self.result_dir + '/' + slugify(comb)
            try:
                os.mkdir(comb_dir)
            except:
                pass

            fname = self.create_file(comb['size'])

            timer = Timer()

            if comb['db_if'] == 'sdk':
                self.upload_file_sdk(client, fname, fname.split('/')[-1])
                up_time = timer.elapsed()
                self.download_file_sdk(client, fname.split('/')[-1],
                                       comb_dir + fname.split('/')[-1])
                dl_time = timer.elapsed() - up_time
                sweeper.done(comb)
            elif comb['db_if'] == 'rest':
                logger.warning('REST interface not implemented')
                sweeper.skip(comb)
                continue
            os.remove(fname)
            f.write("%f %i %f %f \n" % (timer.start_date(), comb['size'],
                                        up_time, dl_time))
        f.close()
    def run(self):
        """ """
        token = 'bRIJb9jp5igAAAAAAAAACc5QzQ619Vp0pYa2PdIrt0q2y0qFyJgwrKvtzuTp3Sz_'
        client = dropbox.client.DropboxClient(token)
        parameters = {'size': igeom(128, 2048, 5), 'db_if': ['rest', 'sdk']}
        combs = sweep(parameters)
        sweeper = ParamSweeper(self.result_dir + "/sweeps", combs)

        f = open(self.result_dir + '/results.txt', 'w')
        while len(sweeper.get_remaining()) > 0:
            comb = sweeper.get_next()

            logger.info('Treating combination %s', pformat(comb))
            comb_dir = self.result_dir + '/' + slugify(comb)
            try:
                os.mkdir(comb_dir)
            except:
                pass

            fname = self.create_file(comb['size'])

            timer = Timer()

            if comb['db_if'] == 'sdk':
                self.upload_file_sdk(client, fname, fname.split('/')[-1])
                up_time = timer.elapsed()
                self.download_file_sdk(client,
                                       fname.split('/')[-1],
                                       comb_dir + fname.split('/')[-1])
                dl_time = timer.elapsed() - up_time
                sweeper.done(comb)
            elif comb['db_if'] == 'rest':
                logger.warning('REST interface not implemented')
                sweeper.skip(comb)
                continue
            os.remove(fname)
            f.write("%f %i %f %f \n" %
                    (timer.start_date(), comb['size'], up_time, dl_time))
        f.close()
Example #11
0
def install_os(reconfigure, tags=None):
    update_config_state()

    # Clone or pull Kolla
    if os.path.isdir('kolla'):
        logger.info("Remove previous Kolla installation")
        kolla_path = os.path.join(SCRIPT_PATH, "kolla")
        call("rm -rf %s" % kolla_path, shell=True)

    logger.info("Cloning Kolla")
    call("cd %s ; git clone %s -b %s > /dev/null" %
         (SCRIPT_PATH, KOLLA_REPO, KOLLA_BRANCH),
         shell=True)

    logger.warning("Patching kolla, this should be \
            deprecated with the new version of Kolla")

    playbook = os.path.join(SCRIPT_PATH, "ansible/patches.yml")
    inventory_path = os.path.join(SYMLINK_NAME, 'multinode')
    run_ansible([playbook], inventory_path, STATE['config'])

    kolla_cmd = [os.path.join(SCRIPT_PATH, "kolla", "tools", "kolla-ansible")]

    if reconfigure:
        kolla_cmd.append('reconfigure')
    else:
        kolla_cmd.append('deploy')

    kolla_cmd.extend([
        "-i",
        "%s/multinode" % SYMLINK_NAME, "--configdir",
        "%s" % SYMLINK_NAME
    ])

    if tags is not None:
        kolla_cmd.extend(["--tags", args])

    call(kolla_cmd)
    def run(self):
        sweeper = self.create_paramsweeper()

        while True:
            comb = sweeper.get_next()
            if not comb:
                break
            comb_dir = self.result_dir + '/' + slugify(comb)
            if not os.path.isdir(comb_dir):
                os.mkdir(comb_dir)
            comb_file = comb_dir + '/trace'
            g5k_configuration['kadeploy3'] = comb['version']
            logger.info('Treating combination %s', pformat(comb))
            get_version = SshProcess(
                comb['version'] + ' -v',
                comb['site'],
                connection_params=default_frontend_connection_params).run()
            logger.info(get_version.stdout)

            resources = ""
            if comb['kavlan']:
                resources += "{type='kavlan'}/vlan=1+"
            resources += "nodes=" + str(comb['n_nodes'])
            sub = OarSubmission(resources=resources,
                                job_type='deploy',
                                walltime="0:30:00",
                                name='Kadeploy_Tests')
            logger.info('Performing submission of %s on site %s', resources,
                        comb['site'])
            jobs = oarsub([(sub, comb['site'])])

            if jobs[0][0]:
                try:
                    logger.info('Waiting for job to start')
                    wait_oar_job_start(jobs[0][0], jobs[0][1])
                    hosts = get_oar_job_nodes(jobs[0][0], jobs[0][1])
                    logger.info('Deployment of %s',
                                ' '.join([host.address for host in hosts]))
                    kavlan = get_oar_job_kavlan(jobs[0][0], jobs[0][1])
                    if kavlan:
                        logger.info('In kavlan %s', kavlan)
                    deployment = Deployment(hosts,
                                            env_name=comb['env'],
                                            vlan=kavlan)
                    deployed, undeployed = deploy(deployment,
                                                  stdout_handlers=[comb_file],
                                                  stderr_handlers=[comb_file])

                finally:
                    logger.info('Destroying job %s on %s', str(jobs[0][0]),
                                jobs[0][1])
                    oardel([(jobs[0][0], jobs[0][1])])
            else:
                deployed = []

            if len(undeployed) == 0:
                logger.info('%s is OK', slugify(comb))
            elif len(deployed) == 0:
                logger.error('%s is KO', slugify(comb))
            else:
                logger.warning('%s encountered problems with some hosts',
                               slugify(comb))

            sweeper.done(comb)
Example #13
0
File: bench.py Project: hyptos/kyd
    def run(self):
        """ run method from engine in order to do our workflow """

        mongo = ClientMongo()

        size = dict
        if not self.options.file:
            if not self.options.only:
                size = {
                    1,
                    long(self.options.size * 0.25),
                    long(self.options.size * 0.5),
                    long(self.options.size * 0.75),
                    long(self.options.size)
                }
            else:
                size = {long(self.options.size)}
        else:
            if self.OnlyDownload:
                size = getFilSize(self.options.file)
            else:
                size = {0}

        drive = None
        if self.options.drive:
            drive = self.options.drive
        else:
            drive = self.drive

        interface = ['rest', 'sdk']
        parameters = {
            'size': size,
            'if': interface,
            'drive': drive,
            'transfert': self.transfert
        }

        p = None

        for n in range(0, int(self.options.ntest), 1):
            logger.info('---------------------')
            logger.info('Round %i', n + 1)
            combs = sweep(parameters)
            date = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
            pathResults = os.getcwd() + '/Results/Bench' + date
            sweeper = ParamSweeper(pathResults + "/sweeps", combs)
            f = open(pathResults + '/results.txt', 'w')

            while len(sweeper.get_remaining()) > 0:
                # sort the parameters
                for i in interface:
                    for dr in drive:
                        for s in size:
                            comb = sweeper.get_next(filtr=lambda r: filter(
                                lambda x: x['drive'] == dr and x['size'] == s
                                and x['if'] == i, r))
                            if not comb: continue

                            # start of the workflow
                            if comb['drive'] == 'amazon':
                                p = providerS3.ProviderS3()
                            elif comb['drive'] == 'dropbox':
                                p = providerDB.ProviderDB()
                            else:
                                p = providerGD.ProviderGD()

                            logger.info('Treating combination %s',
                                        pformat(comb))
                            comb_dir = pathResults + '/' + slugify(comb)

                            if not os.path.isdir(comb_dir):
                                os.mkdir(comb_dir)

                            if not self.options.file:
                                fname = self.create_file(comb['size'])
                            else:
                                fname = self.options.file

                            timer = Timer()
                            up_time = 0
                            dl_time = 0
                            start_date = datetime.datetime.now()
                            if comb['if'] == 'sdk':
                                if p.provider_name == "amazon":
                                    # AMAZON
                                    clientAmz = p.getConnexion()
                                    if self.OnlyDownload:
                                        p.bucketKey += fname
                                    else:
                                        p.bucketKey += '/' + fname
                                    if comb['transfert'] == "upload" or comb[
                                            'transfert'] == 'upDown':
                                        p.upload_file_sdk(
                                            clientAmz.get_bucket(p.bucketName),
                                            p.bucketKey, fname)
                                    up_time = timer.elapsed()

                                    if comb['transfert'] == "download" or comb[
                                            'transfert'] == 'upDown':
                                        p.download_file_sdk(
                                            clientAmz.get_bucket(p.bucketName),
                                            p.bucketKey, comb_dir + '/' +
                                            fname.split('/')[-1])
                                    dl_time = timer.elapsed() - up_time

                                    if not self.OnlyDownload:
                                        p.delete_file_sdk(
                                            clientAmz.get_bucket(p.bucketName),
                                            p.bucketKey)

                                elif p.provider_name == "dropbox":
                                    # DROPBOX
                                    client = p.getToken()
                                    if comb['transfert'] == "upload" or comb[
                                            'transfert'] == 'upDown':
                                        p.upload_file_sdk(
                                            client, fname,
                                            fname.split('/')[-1])
                                    up_time = timer.elapsed()
                                    if comb['transfert'] == "download" or comb[
                                            'transfert'] == 'upDown':
                                        p.download_file_sdk(
                                            client,
                                            fname.split('/')[-1], comb_dir +
                                            '/' + fname.split('/')[-1])
                                    dl_time = timer.elapsed() - up_time

                                    if not self.OnlyDownload:
                                        p.delete_file(client,
                                                      fname.split('/')[-1])

                                elif p.provider_name == "googledrive":
                                    # GOOGLEDRIVE
                                    drive_service = p.getConnexion()
                                    new_file = None
                                    if comb['transfert'] == 'upload' or comb[
                                            'transfert'] == 'upDown':
                                        new_file = p.upload_file_sdk(
                                            drive_service, fname,
                                            fname.split('/')[-1], 'text/plain')
                                        up_time = timer.elapsed()
                                    if comb['transfert'] == 'download' or comb[
                                            'transfert'] == 'upDown':
                                        p.download_file_sdk(
                                            drive_service, new_file, comb_dir +
                                            '/' + fname.split('/')[-1])
                                        dl_time = timer.elapsed() - up_time

                                    if not self.OnlyDownload:
                                        p.delete_file_sdk(
                                            drive_service, new_file['id'])

                                sweeper.done(comb)
                            elif comb['if'] == 'rest':
                                logger.warning(
                                    'REST interface not implemented')
                                sweeper.skip(comb)
                                if not self.OnlyDownload:
                                    # logger.info('delete de '+fname)
                                    if os.path.isfile(fname):
                                        os.remove(fname)
                                        # delete only if rest is implmented
                                        # os.remove(comb_dir + '/' + fname.split('/')[-1])
                                continue
                            if comb['transfert'] == "upload" or comb[
                                    'transfert'] == "upDown":
                                f.write("%s %s %s %s %s %s %s %f %i %s %f\n" %
                                        (self.localisation['ip'],
                                         self.localisation['lat'],
                                         self.localisation['lon'],
                                         self.localisation['city'],
                                         self.localisation['country'],
                                         comb['drive'], comb['if'],
                                         timer.start_date(), comb['size'],
                                         "upload", up_time))
                                mongo.collection.insert({
                                    'ip':
                                    self.localisation['ip'],
                                    'latitude':
                                    self.localisation['lat'],
                                    'longitude':
                                    self.localisation['lon'],
                                    'city':
                                    self.localisation['city'],
                                    'country':
                                    self.localisation['country'],
                                    'drive':
                                    comb['drive'],
                                    'interface':
                                    comb['if'],
                                    'start_date':
                                    start_date,
                                    'size':
                                    comb['size'],
                                    'transfert':
                                    'upload',
                                    'time':
                                    up_time
                                })

                            if comb['transfert'] == "download" or comb[
                                    'transfert'] == "upDown":
                                f.write("%s %s %s %s %s %s %s %f %i %s %f\n" %
                                        (self.localisation['ip'],
                                         self.localisation['lat'],
                                         self.localisation['lon'],
                                         self.localisation['city'],
                                         self.localisation['country'],
                                         comb['drive'], comb['if'],
                                         timer.start_date(), comb['size'],
                                         "download", dl_time))
                                mongo.collection.insert({
                                    'ip':
                                    self.localisation['ip'],
                                    'latitude':
                                    self.localisation['lat'],
                                    'longitude':
                                    self.localisation['lon'],
                                    'city':
                                    self.localisation['city'],
                                    'country':
                                    self.localisation['country'],
                                    'drive':
                                    comb['drive'],
                                    'interface':
                                    comb['if'],
                                    'start_date':
                                    start_date,
                                    'size':
                                    comb['size'],
                                    'transfert':
                                    'download',
                                    'time':
                                    dl_time
                                })

                            if not self.OnlyDownload:
                                # logger.info('delete de '+fname)
                                if os.path.isfile(fname):
                                    os.remove(fname)
                                if os.path.isfile(comb_dir + '/' + fname):
                                    os.remove(comb_dir + '/' +
                                              fname.split('/')[-1])
            f.close()
        # delete the Bench Folder
        os.rmdir(self.result_dir)
        logger.info("---------------------------------------")
        for t in check_Exp_database(self.options, self.localisation)['result']:
            logger.info(t)
Example #14
0
    def workflow(self, comb):
        """
            Compute one application launch 
            using a given parameter group
        """
        comb_ok = False
        try:
            # Generate configuration file needed by MPI processes
            logger.info("Generating assembly file...")
            py = comb['cores'] / comb['px']
            prepare = Process(
                'cd %s && python %s %d %d %d %d %d %s app.lad' %
                (self.workingPath, self.genLadScript, comb['datasize'],
                 comb['datasize'], comb['datasize'], comb['px'], py,
                 comb['transposition']))
            prepare.shell = True
            prepare.run()

            # Generate the MPI host file
            mfile = self.generate_machine_file()

            # Start L2C
            lad = "./app.lad"
            logger.info("Computing...")
            res = Process(
                "export OAR_JOB_KEY_FILE=~/.oar_key ; cd %s && l2c_loader -M,-machinefile,%s --mpi -c %d %s"
                % (self.workingPath, mfile, comb['cores'], lad))
            res.shell = True
            res.stdout_handlers.append(
                os.path.join(self.result_dir,
                             slugify(comb) + '.out'))
            res.stdout_handlers.append(sys.stdout)
            res.stderr_handlers.append(
                os.path.join(self.result_dir,
                             slugify(comb) + '.err'))
            res.stderr_handlers.append(sys.stderr)
            res.run()
            if not res.ok:
                logger.error('Bad L2C termination')
                raise Exception('Bad L2C termination')
            if len(
                    res.stderr
            ) > 0:  # WARNING: when L2C cannot find the LAD file or something strange like this
                logger.warning('Not empty error output')

            # Clean configuration files
            logger.info("Removing assembly files...")
            res = Process('cd %s && rm -f app.lad*' % self.workingPath)
            res.shell = True
            res.run()

            comb_ok = True
        except Exception:
            pass
        finally:
            if comb_ok:
                self.sweeper.done(comb)
                logger.info(style.host(slugify(comb)) + ' has been done')
            else:
                self.sweeper.cancel(comb)
                logger.warning(
                    style.host(slugify(comb)) + ' has been canceled')

            logger.info(style.step('%s Remaining'),
                        len(self.sweeper.get_remaining()))
    def workflow(self, comb, host, comb_dir):
        """ """
        comb_ok = False
        thread_name = style.Thread(str(host).split('.')[0]) + ': '
        logger.info(thread_name + 'Starting combination ' + slugify(comb))

        try:
            self.export = "source ~/aevol_binary/intel/linux/mkl/bin/mklvars.sh intel64; "

            bucketname = self.working_dir + '/raevol_5_mut_lat/' + slugify(
                comb) + '/'

            if os.path.isdir(bucketname) and os.path.exists(bucketname +
                                                            '/last_gener.txt'):
                logger.info(thread_name + "Resuming AEVOL from NFS backup")

                gen_file = open(bucketname + '/last_gener.txt', 'r')

                last_gen = gen_file.read()

                if int(last_gen) < 500000:
                    logger.info(thread_name + "Resuming AEVOL Run from " +
                                str(int(last_gen)))
                    rem = Remote(
                        self.export + 'cd ' + bucketname +
                        '; /home/jorouzaudcornabas/aevol_binary/aevol/src/aevol_run -p 16'
                        + ' -e 300000 -r ' + last_gen + ' >> aevol_run.log',
                        [host]).run()
                    if rem.ok:
                        comb_ok = True
                else:
                    comb_ok = True
            else:
                Remote('mkdir -p ' + bucketname, [host]).run()

                param_file = '/home/jorouzaudcornabas/aevol_binary/execo/mut_lat/param_tmpl.in'

                logger.info(thread_name + 'Generate config file ' + param_file)

                f_template = open(param_file)
                fd, outfile = mkstemp(dir='/tmp/',
                                      prefix=slugify(comb) + '_param')
                f = os.fdopen(fd, 'w')

                for line in f_template:
                    line = line.replace('SEED_NUMBER', str(comb['seed']))
                    line = line.replace('FUZZY_VERSION', str(comb['fuzzy']))
                    if comb['move']:
                        line = line.replace('FIRST_GAUSSIAN_MEDIAN', '0.25')
                        line = line.replace('THIRD_GAUSSIAN_MEDIAN', '0.65')
                    else:
                        line = line.replace('FIRST_GAUSSIAN_MEDIAN', '0.2')
                        line = line.replace('THIRD_GAUSSIAN_MEDIAN', '0.6')
                    line = line.replace('GAUSSIAN_HEIGHT', str(comb['height']))
                    f.write(line)

                f_template.close()
                f.close()

                put_file = Put([host], [outfile],
                               remote_location=bucketname).run()
                if not put_file.ok:
                    exit()

                os.remove(outfile)

                Remote(
                    'cd ' + bucketname + '; cp ' + outfile.split('/')[-1] +
                    ' param.in', [host]).run()

                logger.info(thread_name + "Launching AEVOL Create")
                Remote(
                    self.export + 'cd ' + bucketname +
                    '; /home/jorouzaudcornabas/aevol_diff_area/aevol/src/aevol_create > aevol_create.log',
                    [host]).run()

                logger.info(thread_name + "Launching AEVOL Run")
                rem = Remote(
                    self.export + 'cd ' + bucketname +
                    '; /home/jorouzaudcornabas/aevol_diff_area/aevol/src/aevol_run -p 16 -n 500000 > aevol_run.log',
                    [host]).run()
                if rem.ok:
                    comb_ok = True

            logger.info(thread_name + 'Get results ' + comb_dir + "/" +
                        slugify(comb))

        #try:
        #os.mkdir(comb_dir + "/" + slugify(comb))
        #except:
        #logger.warning(thread_name +
        #'%s already exists, removing existing files', comb_dir + "/" + slugify(comb))

#shutil.rmtree(comb_dir+ "/" + slugify(comb))
#try:
#os.mkdir(comb_dir + "/" + slugify(comb))
#except:
#logger.warning(thread_name +
#'%s already exists, recreating directory', comb_dir + "/" + slugify(comb))

#get_results = Get([host], [bucketname+ "/aevol_create.log", bucketname+ "/aevol_run.log", bucketname+'/stats/'],
#local_location=comb_dir + "/" + slugify(comb)).run()

#for p in get_results.processes:
#if not p.ok:
#logger.error(thread_name +
#': Unable to retrieve the files for combination %s',
#slugify(comb))
#exit()

        finally:
            if comb_ok:
                self.sweeper.done(comb)
                # shutil.rmtree(bucketname)
                logger.info(thread_name + ': ' + slugify(comb) + \
                             ' has been done')
            else:
                self.sweeper.cancel(comb)
                logger.warning(thread_name + ': ' + slugify(comb) + \
                            ' has been canceled')
        logger.info(style.step('%s Remaining'),
                    len(self.sweeper.get_remaining()))
Example #16
0
    def workflow(self, comb, host, comb_dir):
        """ """
        comb_ok = False
        thread_name = style.Thread(str(host).split('.')[0]) + ': '
        logger.info(thread_name + 'Starting combination ' + slugify(comb))
        if 'parapluie' in str(host):
            nb_proc = 24
        elif 'paranoia' in str(host):
            nb_proc = 20
        elif 'parapide' in str(host):
            nb_proc = 8
        else:
            nb_proc = 16

        try:
            self.export = "source ~/aevol_binary/intel/linux/mkl/bin/mklvars.sh intel64; "

            bucketname = self.working_dir + '/raevol_5_mut_lat/' + slugify(
                comb) + '/'

            logger.info(thread_name + "Killing other RAevol")

            killa = Remote("killall -9 aevol_run", [host])
            for killp in killa.processes:
                killp.ignore_error = True
            killa.run()

            if os.path.isdir(bucketname) and os.path.exists(bucketname +
                                                            '/last_gener.txt'):
                logger.info(thread_name + "Resuming AEVOL from NFS backup")

                gen_file = open(bucketname + '/last_gener.txt', 'r')

                last_gen = gen_file.read()

                if int(last_gen) < 300000:
                    logger.info(thread_name + "Resuming AEVOL Run from " +
                                str(int(last_gen)))
                    rem = Remote(
                        self.export + 'cd ' + bucketname +
                        '; /home/jorouzaudcornabas/aevol_binary/aevol/src/aevol_run -p '
                        + str(nb_proc) + ' -e 300000 -r ' + last_gen +
                        ' >> aevol_run.log', [host],
                        process_args={
                            'default_stdout_handler': False,
                            'default_stderr_handler': False
                        }).run()
                    if rem.ok:
                        comb_ok = True
                else:
                    comb_ok = True
            else:
                Remote('mkdir -p ' + bucketname, [host]).run()

                param_file = '/home/jorouzaudcornabas/aevol_binary/aevol/execo/mut_lat/param_tmpl.in'

                logger.info(thread_name + 'Generate config file ' + param_file)

                f_template = open(param_file)
                fd, outfile = mkstemp(dir='/tmp/',
                                      prefix=slugify(comb) + '_param')
                f = os.fdopen(fd, 'w')

                for line in f_template:
                    if 'CONFIGURE_ENVIRONMENT_VALUES' in line:
                        if comb['env'] == 'const':
                            line = line.replace('CONFIGURE_ENVIRONMENT_VALUES',
                                                'NB_ENVIRONMENTS 1')
                            f.write(line)
                            f.write('ENV_ADD_GAUSSIAN  1  0.5   0.2   0.05' +
                                    os.linesep)
                            f.write('ENV_ADD_GAUSSIAN  1  0.5   0.4   0.05' +
                                    os.linesep)
                            f.write('ENV_ADD_GAUSSIAN  1  0.5   0.6   0.05' +
                                    os.linesep)
                            f.write('ENV_ADD_GAUSSIAN  1  0.5   0.8   0.05' +
                                    os.linesep)
                        elif comb['env'] == 'lat_3':
                            line = line.replace('CONFIGURE_ENVIRONMENT_VALUES',
                                                'NB_ENVIRONMENTS 2')
                            f.write(line)
                            f.write('ENV_ADD_GAUSSIAN  1  0.5   0.2   0.05' +
                                    os.linesep)
                            f.write('ENV_ADD_GAUSSIAN  1  0.5   0.4   0.05' +
                                    os.linesep)
                            f.write('ENV_ADD_GAUSSIAN  1  0.5   0.6   0.05' +
                                    os.linesep)
                            f.write('ENV_ADD_GAUSSIAN  1  0.5   0.8   0.05' +
                                    os.linesep)

                            f.write('ENV_ADD_GAUSSIAN  2  0.5   0.2   0.05' +
                                    os.linesep)
                            f.write('ENV_ADD_GAUSSIAN  2  0.5   0.4   0.05' +
                                    os.linesep)
                            f.write('ENV_ADD_GAUSSIAN  2  0.5   0.65  0.05' +
                                    os.linesep)
                            f.write('ENV_ADD_GAUSSIAN  2  0.5   0.8   0.05' +
                                    os.linesep)
                        elif comb['env'] == 'lat_all':
                            line = line.replace('CONFIGURE_ENVIRONMENT_VALUES',
                                                'NB_ENVIRONMENTS 16')
                            f.write(line)

                            #const

                            f.write('ENV_ADD_GAUSSIAN  1  0.5   0.2   0.05' +
                                    os.linesep)
                            f.write('ENV_ADD_GAUSSIAN  1  0.5   0.4   0.05' +
                                    os.linesep)
                            f.write('ENV_ADD_GAUSSIAN  1  0.5   0.6   0.05' +
                                    os.linesep)
                            f.write('ENV_ADD_GAUSSIAN  1  0.5   0.8   0.05' +
                                    os.linesep)

                            # 1

                            f.write('ENV_ADD_GAUSSIAN  2  0.5   0.25  0.05' +
                                    os.linesep)
                            f.write('ENV_ADD_GAUSSIAN  2  0.5   0.4   0.05' +
                                    os.linesep)
                            f.write('ENV_ADD_GAUSSIAN  2  0.5   0.6   0.05' +
                                    os.linesep)
                            f.write('ENV_ADD_GAUSSIAN  2  0.5   0.8   0.05' +
                                    os.linesep)

                            f.write('ENV_ADD_GAUSSIAN  3  0.5   0.2   0.05' +
                                    os.linesep)
                            f.write('ENV_ADD_GAUSSIAN  3  0.5   0.45  0.05' +
                                    os.linesep)
                            f.write('ENV_ADD_GAUSSIAN  3  0.5   0.6   0.05' +
                                    os.linesep)
                            f.write('ENV_ADD_GAUSSIAN  3  0.5   0.8   0.05' +
                                    os.linesep)

                            f.write('ENV_ADD_GAUSSIAN  4  0.5   0.2   0.05' +
                                    os.linesep)
                            f.write('ENV_ADD_GAUSSIAN  4  0.5   0.4   0.05' +
                                    os.linesep)
                            f.write('ENV_ADD_GAUSSIAN  4  0.5   0.65  0.05' +
                                    os.linesep)
                            f.write('ENV_ADD_GAUSSIAN  4  0.5   0.8   0.05' +
                                    os.linesep)

                            f.write('ENV_ADD_GAUSSIAN  5  0.5   0.2   0.05' +
                                    os.linesep)
                            f.write('ENV_ADD_GAUSSIAN  5  0.5   0.4   0.05' +
                                    os.linesep)
                            f.write('ENV_ADD_GAUSSIAN  5  0.5   0.6   0.05' +
                                    os.linesep)
                            f.write('ENV_ADD_GAUSSIAN  5  0.5   0.85  0.05' +
                                    os.linesep)

                            # 2

                            f.write('ENV_ADD_GAUSSIAN  6  0.5   0.25  0.05' +
                                    os.linesep)
                            f.write('ENV_ADD_GAUSSIAN  6  0.5   0.45  0.05' +
                                    os.linesep)
                            f.write('ENV_ADD_GAUSSIAN  6  0.5   0.6   0.05' +
                                    os.linesep)
                            f.write('ENV_ADD_GAUSSIAN  6  0.5   0.8   0.05' +
                                    os.linesep)

                            f.write('ENV_ADD_GAUSSIAN  7  0.5   0.25  0.05' +
                                    os.linesep)
                            f.write('ENV_ADD_GAUSSIAN  7  0.5   0.4   0.05' +
                                    os.linesep)
                            f.write('ENV_ADD_GAUSSIAN  7  0.5   0.65  0.05' +
                                    os.linesep)
                            f.write('ENV_ADD_GAUSSIAN  7  0.5   0.8   0.05' +
                                    os.linesep)

                            f.write('ENV_ADD_GAUSSIAN  8  0.5   0.25  0.05' +
                                    os.linesep)
                            f.write('ENV_ADD_GAUSSIAN  8  0.5   0.4   0.05' +
                                    os.linesep)
                            f.write('ENV_ADD_GAUSSIAN  8  0.5   0.6   0.05' +
                                    os.linesep)
                            f.write('ENV_ADD_GAUSSIAN  8  0.5   0.85  0.05' +
                                    os.linesep)

                            f.write('ENV_ADD_GAUSSIAN  9  0.5   0.2   0.05' +
                                    os.linesep)
                            f.write('ENV_ADD_GAUSSIAN  9  0.5   0.45  0.05' +
                                    os.linesep)
                            f.write('ENV_ADD_GAUSSIAN  9  0.5   0.65  0.05' +
                                    os.linesep)
                            f.write('ENV_ADD_GAUSSIAN  9  0.5   0.8   0.05' +
                                    os.linesep)

                            f.write('ENV_ADD_GAUSSIAN  10  0.5   0.2   0.05' +
                                    os.linesep)
                            f.write('ENV_ADD_GAUSSIAN  10  0.5   0.45  0.05' +
                                    os.linesep)
                            f.write('ENV_ADD_GAUSSIAN  10  0.5   0.6   0.05' +
                                    os.linesep)
                            f.write('ENV_ADD_GAUSSIAN  10  0.5   0.85  0.05' +
                                    os.linesep)

                            f.write('ENV_ADD_GAUSSIAN  11  0.5   0.2   0.05' +
                                    os.linesep)
                            f.write('ENV_ADD_GAUSSIAN  11  0.5   0.4   0.05' +
                                    os.linesep)
                            f.write('ENV_ADD_GAUSSIAN  11  0.5   0.65  0.05' +
                                    os.linesep)
                            f.write('ENV_ADD_GAUSSIAN  11  0.5   0.85  0.05' +
                                    os.linesep)

                            # 3

                            f.write('ENV_ADD_GAUSSIAN  12  0.5   0.25  0.05' +
                                    os.linesep)
                            f.write('ENV_ADD_GAUSSIAN  12  0.5   0.45  0.05' +
                                    os.linesep)
                            f.write('ENV_ADD_GAUSSIAN  12  0.5   0.65  0.05' +
                                    os.linesep)
                            f.write('ENV_ADD_GAUSSIAN  12  0.5   0.8   0.05' +
                                    os.linesep)

                            f.write('ENV_ADD_GAUSSIAN  13  0.5   0.25  0.05' +
                                    os.linesep)
                            f.write('ENV_ADD_GAUSSIAN  13  0.5   0.45  0.05' +
                                    os.linesep)
                            f.write('ENV_ADD_GAUSSIAN  13  0.5   0.6   0.05' +
                                    os.linesep)
                            f.write('ENV_ADD_GAUSSIAN  13  0.5   0.85  0.05' +
                                    os.linesep)

                            f.write('ENV_ADD_GAUSSIAN  14  0.5   0.25  0.05' +
                                    os.linesep)
                            f.write('ENV_ADD_GAUSSIAN  14  0.5   0.4   0.05' +
                                    os.linesep)
                            f.write('ENV_ADD_GAUSSIAN  14  0.5   0.65  0.05' +
                                    os.linesep)
                            f.write('ENV_ADD_GAUSSIAN  14  0.5   0.85  0.05' +
                                    os.linesep)

                            f.write('ENV_ADD_GAUSSIAN  15  0.5   0.2   0.05' +
                                    os.linesep)
                            f.write('ENV_ADD_GAUSSIAN  15  0.5   0.45  0.05' +
                                    os.linesep)
                            f.write('ENV_ADD_GAUSSIAN  15  0.5   0.65  0.05' +
                                    os.linesep)
                            f.write('ENV_ADD_GAUSSIAN  15  0.5   0.85  0.05' +
                                    os.linesep)

                            # 4

                            f.write('ENV_ADD_GAUSSIAN  16  0.5   0.25  0.05' +
                                    os.linesep)
                            f.write('ENV_ADD_GAUSSIAN  16  0.5   0.45  0.05' +
                                    os.linesep)
                            f.write('ENV_ADD_GAUSSIAN  16  0.5   0.65  0.05' +
                                    os.linesep)
                            f.write('ENV_ADD_GAUSSIAN  16  0.5   0.85  0.05' +
                                    os.linesep)
                    elif 'CONFIGURE_SIGNAL_VALUES' in line:
                        if comb['env'] == 'const':
                            line = line.replace('CONFIGURE_SIGNAL_VALUES', '')
                            f.write(line)

                        elif comb['env'] == 'lat_3':
                            line = line.replace(
                                'CONFIGURE_SIGNAL_VALUES',
                                'CREATE_SIGNAL h0 h0 h0 w0 m0 m1 m0 h1 h0 m0 h0 m1 h1 w0 h1 h0 m1 h1 m0 w0 w0 m0 w0 h0 h1 m1 w0 m0 m1 m0 w0 h1 h0 m0 h0 m1 h1 w0 h0 w0 m0 m1 m0 w0 h1 h0 w0 w0 h1'
                            )
                            f.write(line)

                            f.write('ENV_ADD_SIGNAL 2 1' + os.linesep)
                        elif comb['env'] == 'lat_all':
                            line = line.replace(
                                'CONFIGURE_SIGNAL_VALUES',
                                'CREATE_SIGNAL h0 w0 h1 m1 w0 h1 m0 h0 h1 w0 h0 m1 h1 h1 m1 m0 h0 w0 h1 m1 w0 h1 m0 h0 h1 w0 h0 m1 h1 h1 m1 m0 h1 m0 m1'
                            )
                            f.write(line)
                            f.write(
                                'CREATE_SIGNAL m0 h0 m1 h1 m1 w0 m0 m1 m0 h0 m1 h1 w0 h0 h0 h1 m1 m0 h1 w0 h1 h0 m1 h1 m0 w0 w0 m0 m1 w0 w0 h1 h0 w0 h1 h0 h0 m0 h0 w0 h0 m1 m0 w0 h1 w0 w0 h1 m0'
                                + os.linesep)
                            f.write(
                                'CREATE_SIGNAL h0 h0 h0 w0 m0 m1 m0 h1 h0 m0 h0 m1 h1 w0 h1 h0 m1 h1 m0 w0 w0 m0 w0 h0 h1 m1 w0 m0 m1 m0 w0 h1 h0 m0 h0 m1 h1 w0 h0 w0 m0 m1 m0 w0 h1 h0 w0 w0 h1'
                                + os.linesep)
                            f.write(
                                'CREATE_SIGNAL h1 h1 m0 w0 w0 h1 m1 h1 h1 m1 m0 w0 m1 m0 m0 w0 m0 h0 m0 h0 w0 h0 m0 h0 h1 m1 h0 h1 w0 h0 h1 m1 h1 m1 m0'
                                + os.linesep)

                            f.write('ENV_ADD_SIGNAL 2 1' + os.linesep)
                            f.write('ENV_ADD_SIGNAL 3 2' + os.linesep)
                            f.write('ENV_ADD_SIGNAL 4 3' + os.linesep)
                            f.write('ENV_ADD_SIGNAL 5 4' + os.linesep)

                            f.write('ENV_ADD_SIGNAL 6 1' + os.linesep)
                            f.write('ENV_ADD_SIGNAL 6 2' + os.linesep)

                            f.write('ENV_ADD_SIGNAL 7 1' + os.linesep)
                            f.write('ENV_ADD_SIGNAL 7 3' + os.linesep)

                            f.write('ENV_ADD_SIGNAL 8 1' + os.linesep)
                            f.write('ENV_ADD_SIGNAL 8 4' + os.linesep)

                            f.write('ENV_ADD_SIGNAL 9 2' + os.linesep)
                            f.write('ENV_ADD_SIGNAL 9 3' + os.linesep)

                            f.write('ENV_ADD_SIGNAL 10 2' + os.linesep)
                            f.write('ENV_ADD_SIGNAL 10 4' + os.linesep)

                            f.write('ENV_ADD_SIGNAL 11 3' + os.linesep)
                            f.write('ENV_ADD_SIGNAL 11 4' + os.linesep)

                            f.write('ENV_ADD_SIGNAL 12 1' + os.linesep)
                            f.write('ENV_ADD_SIGNAL 12 2' + os.linesep)
                            f.write('ENV_ADD_SIGNAL 12 3' + os.linesep)

                            f.write('ENV_ADD_SIGNAL 13 1' + os.linesep)
                            f.write('ENV_ADD_SIGNAL 13 2' + os.linesep)
                            f.write('ENV_ADD_SIGNAL 13 4' + os.linesep)

                            f.write('ENV_ADD_SIGNAL 14 1' + os.linesep)
                            f.write('ENV_ADD_SIGNAL 14 3' + os.linesep)
                            f.write('ENV_ADD_SIGNAL 14 4' + os.linesep)

                            f.write('ENV_ADD_SIGNAL 15 2' + os.linesep)
                            f.write('ENV_ADD_SIGNAL 15 3' + os.linesep)
                            f.write('ENV_ADD_SIGNAL 15 4' + os.linesep)

                            f.write('ENV_ADD_SIGNAL 16 1' + os.linesep)
                            f.write('ENV_ADD_SIGNAL 16 2' + os.linesep)
                            f.write('ENV_ADD_SIGNAL 16 3' + os.linesep)
                            f.write('ENV_ADD_SIGNAL 16 4' + os.linesep)
                    else:
                        line = line.replace('SEED_NUMBER', str(comb['seed']))
                        line = line.replace('MUTATION_RATE_VALUE',
                                            comb['mutation'])
                        line = line.replace('SELECTION_PRESSURE',
                                            str(comb['selection']))
                        f.write(line)

                f_template.close()
                f.close()

                put_file = Put([host], [outfile],
                               remote_location=bucketname).run()
                if not put_file.ok:
                    exit()

                os.remove(outfile)

                Remote(
                    'cd ' + bucketname + '; cp ' + outfile.split('/')[-1] +
                    ' param.in; cp /home/jorouzaudcornabas/aevol_binary/aevol/execo/mut_lat/binding_matrix.rae .',
                    [host]).run()

                logger.info(thread_name + "Launching AEVOL Create")
                Remote(
                    self.export + 'cd ' + bucketname +
                    '; /home/jorouzaudcornabas/aevol_binary/aevol/src/aevol_create > aevol_create.log',
                    [host],
                    process_args={
                        'default_stdout_handler': False,
                        'default_stderr_handler': False
                    }).run()

                logger.info(thread_name + "Launching AEVOL Run")
                rem = Remote(
                    self.export + 'cd ' + bucketname +
                    '; /home/jorouzaudcornabas/aevol_binary/aevol/src/aevol_run -p '
                    + str(nb_proc) + ' -n 300000 > aevol_run.log', [host],
                    process_args={
                        'default_stdout_handler': False,
                        'default_stderr_handler': False
                    }).run()
                if rem.ok:
                    comb_ok = True

            logger.info(thread_name + 'Get results ' + comb_dir + "/" +
                        slugify(comb))

        #try:
        #os.mkdir(comb_dir + "/" + slugify(comb))
        #except:
        #logger.warning(thread_name +
        #'%s already exists, removing existing files', comb_dir + "/" + slugify(comb))

#shutil.rmtree(comb_dir+ "/" + slugify(comb))
#try:
#os.mkdir(comb_dir + "/" + slugify(comb))
#except:
#logger.warning(thread_name +
#'%s already exists, recreating directory', comb_dir + "/" + slugify(comb))

#get_results = Get([host], [bucketname+ "/aevol_create.log", bucketname+ "/aevol_run.log", bucketname+'/stats/'],
#local_location=comb_dir + "/" + slugify(comb)).run()

#for p in get_results.processes:
#if not p.ok:
#logger.error(thread_name +
#': Unable to retrieve the files for combination %s',
#slugify(comb))
#exit()

        finally:
            if comb_ok:
                self.sweeper.done(comb)
                # shutil.rmtree(bucketname)
                logger.info(thread_name + ': ' + slugify(comb) + \
                             ' has been done')
            else:
                self.sweeper.cancel(comb)
                logger.warning(thread_name + ': ' + slugify(comb) + \
                            ' has been canceled')
        logger.info(style.step('%s Remaining'),
                    len(self.sweeper.get_remaining()))
Example #17
0
def prepare_node(conf_file, force_deploy, tags):
    g5k = G5kEngine(conf_file, force_deploy)

    g5k.start(args=[])

    STATE['config'].update(g5k.load())

    g5k.get_job()

    deployed, undeployed = g5k.deploy()
    if len(undeployed) > 0:
        sys.exit(31)

    roles = g5k.build_roles()

    # Get an IP for
    # kolla (haproxy)
    # docker registry
    # influx db
    # grafana
    vip_addresses = g5k.get_free_ip(5)
    # Get the NIC devices of the reserved cluster
    # XXX: this only works if all nodes are on the same cluster,
    # or if nodes from different clusters have the same devices
    interfaces = g5k.get_cluster_nics(STATE['config']['resources'].keys()[0])
    network_interface = str(interfaces[0])
    external_interface = None

    if len(interfaces) > 1:
        external_interface = str(interfaces[1])
    else:
        external_interface = 'veth0'
        logger.warning("%s has only one NIC. The same interface "
                       "will be used for network_interface and "
                       "neutron_external_interface." %
                       STATE['config']['resources'].keys()[0])

    g5k.exec_command_on_nodes(
        g5k.deployed_nodes,
        'apt-get update && apt-get -y --force-yes install apt-transport-https',
        'Installing apt-transport-https...')

    # Install python on the nodes
    g5k.exec_command_on_nodes(g5k.deployed_nodes, 'apt-get -y install python',
                              'Installing Python on all the nodes...')

    # Generates files for ansible/kolla
    inventory_path = os.path.join(g5k.result_dir, 'multinode')
    base_inventory = STATE['config']['inventory']
    generate_inventory(roles, base_inventory, inventory_path)

    # Symlink current directory
    link = os.path.abspath(SYMLINK_NAME)
    try:
        os.remove(link)
    except OSError:
        pass
    os.symlink(g5k.result_dir, link)
    logger.info("Symlinked %s to %s" % (g5k.result_dir, link))

    STATE['config'].update({
        'vip': str(vip_addresses[0]),
        'registry_vip': str(vip_addresses[1]),
        'influx_vip': str(vip_addresses[2]),
        'grafana_vip': str(vip_addresses[3]),
        'network_interface': network_interface
    })

    # Run the Ansible playbooks
    playbook_path = os.path.join(SCRIPT_PATH, 'ansible', 'prepare-node.yml')
    passwords_path = os.path.join(TEMPLATE_DIR, "passwords.yml")
    inventory_path = os.path.join(SYMLINK_NAME, 'multinode')
    config = STATE['config'].copy()
    with open(passwords_path) as passwords_file:
        config.update(yaml.load(passwords_file))

    kolla_vars = {
        'kolla_internal_vip_address': str(vip_addresses[0]),
        'network_interface': network_interface,
        'neutron_external_interface': external_interface,
        'enable_veth': external_interface == 'veth0',
        'neutron_external_address': str(vip_addresses[4])
    }

    config.update(kolla_vars)

    run_ansible([playbook_path], inventory_path, config, tags)

    # Generating Ansible globals.yml, passwords.yml
    generate_kolla_files(g5k.config["kolla"], kolla_vars, g5k.result_dir)

    # Fills the state and save it in the `current` directory
    # TODO: Manage STATE at __main__ level
    STATE['config_file'] = conf_file
    STATE['nodes'] = roles
    STATE['user'] = g5k.user
Example #18
0
def prepare_node(conf_file, force_deploy, tags):
    g5k = G5kEngine(conf_file, force_deploy)

    g5k.start(args=[])

    STATE['config'].update(g5k.load())

    g5k.get_job()

    deployed, undeployed = g5k.deploy()
    if len(undeployed) > 0:
        sys.exit(31)

    roles = g5k.build_roles()

    # Get an IP for
    # kolla (haproxy)
    # docker registry
    # influx db
    # grafana
    vip_addresses = g5k.get_free_ip(5)
    # Get the NIC devices of the reserved cluster
    # XXX: this only works if all nodes are on the same cluster,
    # or if nodes from different clusters have the same devices
    interfaces = g5k.get_cluster_nics(STATE['config']['resources'].keys()[0])
    network_interface = str(interfaces[0])
    external_interface = None

    if len(interfaces) > 1:
        external_interface = str(interfaces[1])
    else:
        external_interface = 'veth0'
        logger.warning("%s has only one NIC. The same interface "
                       "will be used for network_interface and "
                       "neutron_external_interface."
                       % STATE['config']['resources'].keys()[0])


    g5k.exec_command_on_nodes(
        g5k.deployed_nodes,
        'apt-get update && apt-get -y --force-yes install apt-transport-https',
        'Installing apt-transport-https...')

    # Install python on the nodes
    g5k.exec_command_on_nodes(
        g5k.deployed_nodes,
        'apt-get -y install python',
        'Installing Python on all the nodes...')

    # Generates files for ansible/kolla
    inventory_path = os.path.join(g5k.result_dir, 'multinode')
    base_inventory = STATE['config']['inventory']
    generate_inventory(roles, base_inventory, inventory_path)

    # Symlink current directory
    link = os.path.abspath(SYMLINK_NAME)
    try:
        os.remove(link)
    except OSError:
        pass
    os.symlink(g5k.result_dir, link)
    logger.info("Symlinked %s to %s" % (g5k.result_dir, link))

    STATE['config'].update({
        'vip': str(vip_addresses[0]),
        'registry_vip': str(vip_addresses[1]),
        'influx_vip': str(vip_addresses[2]),
        'grafana_vip': str(vip_addresses[3]),
        'network_interface': network_interface
    })

    # Run the Ansible playbooks
    playbook_path = os.path.join(SCRIPT_PATH, 'ansible', 'prepare-node.yml')
    passwords_path = os.path.join(TEMPLATE_DIR, "passwords.yml")
    inventory_path = os.path.join(SYMLINK_NAME, 'multinode')
    config = STATE['config'].copy()
    with open(passwords_path) as passwords_file:
        config.update(yaml.load(passwords_file))

    kolla_vars = {
        'kolla_internal_vip_address' : str(vip_addresses[0]),
        'network_interface'          : network_interface,
        'neutron_external_interface' : external_interface,
        'enable_veth'                : external_interface == 'veth0',
        'neutron_external_address'   : str(vip_addresses[4])
    }

    config.update(kolla_vars)

    run_ansible([playbook_path], inventory_path, config, tags)

    # Generating Ansible globals.yml, passwords.yml
    generate_kolla_files(g5k.config["kolla"], kolla_vars, g5k.result_dir)

    # Fills the state and save it in the `current` directory
    # TODO: Manage STATE at __main__ level
    STATE['config_file'] = conf_file
    STATE['nodes']  = roles
    STATE['user']   = g5k.user
    def run(self):
        sweeper = self.create_paramsweeper()

        while True:
            comb = sweeper.get_next()
            if not comb:
                break
            comb_dir = self.result_dir + '/' + slugify(comb)
            if not os.path.isdir(comb_dir):
                os.mkdir(comb_dir)
            comb_file = comb_dir + '/trace'
            g5k_configuration['kadeploy3'] = comb['version']
            logger.info('Treating combination %s', pformat(comb))
            get_version = SshProcess(comb['version'] + ' -v',
                                     comb['site'],
                                     connection_params=default_frontend_connection_params).run()
            logger.info(get_version.stdout)

            resources = ""
            if comb['kavlan']:
                resources += "{type='kavlan'}/vlan=1+"
            resources += "nodes=" + str(comb['n_nodes'])
            sub = OarSubmission(resources=resources,
                                job_type='deploy',
                                walltime="0:30:00",
                                name='Kadeploy_Tests')
            logger.info('Performing submission of %s on site %s',
                        resources, comb['site'])
            jobs = oarsub([(sub, comb['site'])])

            if jobs[0][0]:
                try:
                    logger.info('Waiting for job to start')
                    wait_oar_job_start(jobs[0][0], jobs[0][1])
                    hosts = get_oar_job_nodes(jobs[0][0], jobs[0][1])
                    logger.info('Deployment of %s',
                                ' '.join([host.address for host in hosts]))
                    kavlan = get_oar_job_kavlan(jobs[0][0], jobs[0][1])
                    if kavlan:
                        logger.info('In kavlan %s', kavlan)
                    deployment = Deployment(hosts, env_name=comb['env'],
                                            vlan=kavlan)
                    deployed, undeployed = deploy(deployment,
                                                  stdout_handlers=[comb_file],
                                                  stderr_handlers=[comb_file])

                finally:
                    logger.info('Destroying job %s on %s', str(jobs[0][0]),
                                jobs[0][1])
                    oardel([(jobs[0][0], jobs[0][1])])
            else:
                deployed = []

            if len(undeployed) == 0:
                logger.info('%s is OK', slugify(comb))
            elif len(deployed) == 0:
                logger.error('%s is KO', slugify(comb))
            else:
                logger.warning('%s encountered problems with some hosts',
                               slugify(comb))

            sweeper.done(comb)