Exemplo n.º 1
0
    def _set_run_dirs(self):
        """
        Sets the relative run directories as instance 
        variables for all runs
        """
        def get_run_number(number):
            return str(number).zfill(5)

        def get_shard_dir(shard_number):
            first_run = self.shard_size * (shard_number - 1) + 1
            last_run = self.shard_size * (shard_number)
            return 'runs-%s-%s' % (get_run_number(first_run),
                                   get_run_number(last_run))

        current_run = 0

        shards = tools.divide_list(self.runs, self.shard_size)

        for shard_number, shard in enumerate(shards, start=1):
            shard_dir = os.path.join(self.base_dir,
                                     get_shard_dir(shard_number))
            tools.overwrite_dir(shard_dir)

            for run in shard:
                current_run += 1
                rel_dir = os.path.join(get_shard_dir(shard_number),
                                       get_run_number(current_run))
                abs_dir = os.path.join(self.base_dir, rel_dir)
                run.dir = abs_dir
Exemplo n.º 2
0
    def _build_main_script(self):
        """
        Generates the main script
        """
        num_tasks = math.ceil(len(self.runs) / float(self.runs_per_task))
        current_dir = os.path.dirname(os.path.abspath(__file__))
        job_params = {
            'logfile':
            os.path.join(current_dir, self.exp_name, self.exp_name + '.log'),
            'errfile':
            os.path.join(current_dir, self.exp_name, self.exp_name + '.err'),
            'driver_timeout':
            self.timeout + 30,
            'num_tasks':
            num_tasks,
            'queue':
            self.queue,
            'priority':
            self.priority,
        }
        script_template = open('data/gkigrid-job-header-template').read()
        script = script_template % job_params

        script += '\n'

        run_groups = tools.divide_list(self.runs, self.runs_per_task)

        for task_id, run_group in enumerate(run_groups, start=1):
            script += 'if [[ $SGE_TASK_ID == %s ]]; then\n' % task_id
            for run in run_group:
                # Change into the run dir
                script += '  cd %s\n' % run.dir
                script += '  ./run\n'
            script += 'fi\n'

        name = self.exp_name
        filename = name if name.endswith('.q') else name + '.q'
        filename = self._get_abs_path(filename)

        with open(filename, 'w') as file:
            file.write(script)
Exemplo n.º 3
0
    def _set_run_dirs(self):
        """
        Sets the relative run directories as instance variables for all runs
        """
        def run_number(number):
            return str(number).zfill(5)

        def get_shard_dir(shard_number):
            first_run = self.shard_size * (shard_number - 1) + 1
            last_run = self.shard_size * (shard_number)
            return 'runs-%s-%s' % (run_number(first_run), run_number(last_run))

        current_run = 0
        shards = tools.divide_list(self.runs, self.shard_size)

        for shard_number, shard in enumerate(shards, start=1):
            shard_dir = os.path.join(self.path, get_shard_dir(shard_number))

            for run in shard:
                current_run += 1
                rel_dir = os.path.join(get_shard_dir(shard_number),
                                       run_number(current_run))
                run.dir = self._get_abs_path(rel_dir)