예제 #1
0
    def control_one_job(self, id):
        """ control the status of a single job with it's cluster id """
        cmd = 'qstat ' + str(id)
        status = misc.Popen([cmd],
                            shell=True,
                            stdout=subprocess.PIPE,
                            stderr=open(os.devnull, 'w'))

        for line in status.stdout:
            line = line.strip()
            if 'Unknown' in line:
                return 'F'
            elif line.startswith(str(id)):
                status = line.split()[4]
        if status in self.idle_tag:
            return 'I'
        elif status in self.running_tag:
            return 'R'
        return 'F'
예제 #2
0
 def control_one_job(self, id):
     """ control the status of a single job with it's cluster id """
     #cmd = 'qstat '+str(id)
     cmd = 'qstat '
     status = misc.Popen([cmd], shell=True, stdout=subprocess.PIPE)
     for line in status.stdout:
         #print "!==",line
         #line = line.strip()
         #if 'Unknown' in line:
         #    return 'F'
         #elif line.startswith(str(id)):
         #    status = line.split()[4]
         if str(id) in line:
             status = line.split()[4]
             #print "!=status", status
     if status in self.idle_tag:
         return 'I'
     elif status in self.running_tag:
         return 'R'
     return 'F'
예제 #3
0
    def control_one_job(self, id):
        """ control the status of a single job with it's cluster id """

        cmd = 'bjobs ' + str(id)
        status = misc.Popen([cmd], shell=True, stdout=subprocess.PIPE)

        for line in status.stdout:
            line = line.strip().upper()
            if 'JOBID' in line:
                continue
            elif str(id) not in line:
                continue
            status = line.split()[2]
            if status == 'RUN':
                return 'R'
            elif status == 'PEND':
                return 'I'
            elif status == 'DONE':
                return 'F'
            else:
                return 'H'
            return 'F'
예제 #4
0
    def control(self, me_dir):
        """ control the status of a single job with it's cluster id """
        cmd = "qstat "
        status = misc.Popen([cmd], shell=True, stdout=subprocess.PIPE)

        if me_dir.endswith('/'):
            me_dir = me_dir[:-1]
        me_dir = hashlib.md5(me_dir).hexdigest()[-10:]
        if not me_dir[0].isalpha():
            me_dir = 'a' + me_dir[1:]

        idle, run, fail = 0, 0, 0
        for line in status.stdout:
            if me_dir in line:
                status = line.split()[4]
                if status in self.idle_tag:
                    idle += 1
                elif status in self.running_tag:
                    run += 1
                else:
                    logger.debug(line)
                    fail += 1

        return idle, run, self.submitted - (idle + run + fail), fail
예제 #5
0
    def submit2(self, prog, argument=[], cwd=None, stdout=None, stderr=None,
            log=None, input_files=[], output_files=[], required_output=[],nb_submit=0):
        """How to make one submission. Return status id on the cluster.
        NO SHARE DISK"""

        #print "running lsf submit2"

        if cwd is None:
            cwd = os.getcwd()
        if not os.path.exists(prog):
            prog = os.path.join(cwd, prog)

        if not required_output and output_files:
            required_output = output_files

        if not self.dorsync or (not input_files and not output_files):
            # not input/output so not using submit2
            return self.submit(prog, argument, cwd, stdout, stderr, log,
        required_output=required_output, nb_submit=nb_submit)

        if self.rsyncd.poll()!=None:
            raise RuntimeError("rsyncd not running")

        if cwd is None:
            cwd = os.getcwd()
        if not os.path.exists(prog):
            prog = os.path.join(cwd, prog)
        temp_file_name = "sub." + os.path.basename(prog) + '.'.join(argument)
               
        input_files.append(prog)                
               
        hostname = platform.node()
               
        rsynccwd = cwd
        if rsynccwd.startswith(self.rsyncroot):
            rsynccwd = rsynccwd[len(self.rsyncroot):]                   
               
        infilelist = ""
        for input_file in input_files:
            #make sure input_files are absolute paths
            if not input_file.startswith('/'):
                input_file = os.path.join(cwd,input_file)
            #convert to paths relative to rsyncd root
            if input_file.startswith(self.rsyncroot):
                input_file = input_file[len(self.rsyncroot):]
            infilelist += "%s@%s::%s/%s " % (self.rsyncuser,hostname,self.rsyncmodule, input_file)
        infilelist += "./"
        
        outfilelist = ""
        for output_file in output_files:
            outfilelist += "%s " % (output_file)
        outfilelist += "%s@%s::%s/%s" % (self.rsyncuser,hostname,self.rsyncmodule,rsynccwd)
            
        text = """#!/bin/bash
        
            SUBMITTERHOST=%(hostname)s            

            if [ -n $CMSSW_VERSION ]
            then
              scramv1 project CMSSW $CMSSW_VERSION
              cd $CMSSW_VERSION
              eval `scramv1 runtime -sh`
              cd -
            fi
                             
            export RSYNC_PASSWORD=$MADGRAPHRSYNCPASSWD_%(rsyncport)s
                 
            #dereference symlinks for input
            rsync -vvv --timeout=600 --contimeout=600 --port %(rsyncport)s -rptL %(infilelist)s

            echo '%(arguments)s' > arguments
            chmod +x ./%(script)s        
            %(program)s ./%(script)s %(arguments)s
            
            #copy symlinks as symlinks for output and don't overwrite existing files unless updated
            rsync -vvv --timeout=600 --contimeout=600 --port %(rsyncport)s -rptul %(outfilelist)s
            
            """
        dico = {'script': os.path.basename(prog),
        'hostname': hostname,
        'infilelist': infilelist,
        'outfilelist': outfilelist,
        'output_files': ' '.join(output_files),
        'rsyncport': self.rsyncport,
        'arguments': ' '.join([str(a) for a in argument]),
        'program': ' ' if '.py' in prog else 'bash'}

        me_dir = self.get_jobs_identifier(cwd, prog)

        text = text % dico
        cwdpath = "/tmp/" + os.environ.get("USER", '')
        command = ['bsub', '-cwd', cwdpath, '-C0', '-J', me_dir]
        if cwd is None:
            cwd = os.getcwd()
        #else:
            #text += " cd %s;" % cwd
        if stdout and isinstance(stdout, str):
            command.extend(['-o', stdout])
        if stderr and isinstance(stdout, str):
            command.extend(['-e', stderr])
        elif stderr == -2: # -2 is subprocess.STDOUT
            pass
        if log is None:
            log = '/dev/null'

        if self.cluster_queue and self.cluster_queue != 'None':
            command.extend(['-q', self.cluster_queue])

        submitenv = os.environ.copy()
        submitenv["TMPDIR"] = "/tmp/" + submitenv.get("USER", '')
        a = misc.Popen(command, stdout=subprocess.PIPE,
        stderr=subprocess.STDOUT,
        stdin=subprocess.PIPE, cwd=cwd,
        env=submitenv)

        output = a.communicate(text)[0]
        #Job <nnnn> is submitted to default queue <normal>.
        try:
            id = output.split('>',1)[0].split('<')[1]
        except:
            raise ClusterManagmentError, 'fail to submit to the cluster: \n%s' \
            % output
        if not id.isdigit():
            raise ClusterManagmentError, 'fail to submit to the cluster: \n%s' \
            % output
        self.submitted += 1
        self.submitted_ids.append(id)
        return id         
예제 #6
0
파일: IOTests.py 프로젝트: mfasDa/MadGraph5
    def runIOTests(self, update = False, force = 0, verbose=False, \
                                                       testKeys='instanceList'):
        """ Run the IOTests for this instance (defined in self.instance_tests)
            and compare the files of the chosen tests against the hardcoded ones
            stored in tests/input_files/IOTestsComparison. If you see
            an error in the comparison and you are sure that the newest output
            is correct (i.e. you understand that this modification is meant to
            be so). Then feel free to automatically regenerate this file with
            the newest version by doing 
            
              ./test_manager -i U folderName/testName/fileName
                
            If update is True (meant to be used by __main__ only) then
            it will create/update/remove the files instead of testing them.
            The argument tests can be a list of tuple-keys describing the tests
            to cover. Otherwise it is the instance_test list.
            The force argument must be 10 if you do not want to monitor the 
            modifications on the updated files. If it is 0 you will monitor
            all modified file and if 1 you will monitor each modified file of
            a given name only once.
        """
        # First make sure that the tarball need not be untarred
        # Extract the tarball for hardcoded in all cases to make sure the
        # IOTestComparison folder is synchronized with it.
        if IOTestManager._compress_ref_fodler:
            if path.isdir(_hc_comparison_files):
                try:
                    shutil.rmtree(_hc_comparison_files)
                except IOError:
                    pass
            if path.isfile(_hc_comparison_tarball):
                tar = tarfile.open(_hc_comparison_tarball, mode='r:bz2')
                tar.extractall(path.dirname(_hc_comparison_files))
                tar.close()
            else:
                raise MadGraph5Error(
                    "Could not find the comparison tarball %s." %
                    _hc_comparison_tarball)
        else:
            if not path.isdir(_hc_comparison_files):
                raise MadGraph5Error(
                    "Could not find the comparison tarball %s." %
                    _hc_comparison_tarball)

        # In update = True mode, we keep track of the modification to
        # provide summary information
        modifications = {
            'updated': [],
            'created': [],
            'removed': [],
            'missing': []
        }

        # List all the names of the files for which modifications have been
        # reviewed at least once.The approach taken here is different than
        # with the list refusedFolder and refusedTest.
        # The key of the dictionary are the filenames and the value are string
        # determining the user answer for that file.
        reviewed_file_names = {}

        # Chose what test to cover
        if testKeys == 'instanceList':
            testKeys = self.instance_tests

        if verbose:            print("\n== "+colored%(32,"Operational mode")+\
    " : file %s ==\n"%(colored%(34,('UPDATE' if update else 'TESTING'))))
        for (folder_name, test_name) in testKeys:
            try:
                (iotest, iotestManager) = self.all_tests[(folder_name,
                                                          test_name)]
            except KeyError:
                raise MadGraph5Error('Test (%s,%s) could not be found.'\
                                                       %(folder_name, test_name))
            if verbose:
                print("Processing %s in %s" %
                      (colored % (32, test_name), colored % (34, folder_name)))

            files_path = iotest.run(iotestManager)
            try:
                pass
#                files_path = iotest.run(iotestManager)
            except Exception as e:
                iotest.clean_output()
                if not verbose:
                    raise e
                else:
                    print(colored%(31,"  Test %s "%test_name+\
                              "crashed with the following error:\n  %s."%str(e)))
                    continue

            # First create the list of files to check as the user might be using
            # regular expressions.
            filesToCheck = []
            # Store here the files reckognized as veto rules (with filename
            # starting with '-')
            veto_rules = []
            for fname in iotest.testedFiles:
                # Disregard the veto rules
                regexp_finder = re.compile(
                    r'^(?P<veto>-)?(?P<root_folder>.*)(\/)?\[(?P<regexp>.*)\]$'
                )
                found = regexp_finder.search(fname)
                if not found is None:
                    # folder without the final /
                    base_path = pjoin(files_path, found.group('root_folder'))
                    regexp = re.compile(found.group('regexp'))
                    # In filesToCheck, we must remove the files_path/ prepended
                    for root, dirnames, filenames in os.walk(base_path):
                        for file in filenames:
                            if not regexp.search(str(os.path.relpath(
                                    pjoin(root,file),base_path))) is None and \
                                              not path.islink(pjoin(root,file)):
                                new_target = os.path.relpath(
                                    pjoin(root, file), files_path)
                                if found.group('veto') == '-':
                                    veto_rules.append(new_target)
                                else:
                                    filesToCheck.append(new_target)
                else:
                    fn = fname[1:] if fname.startswith('-') else fname
                    if (not path.exists(pjoin(files_path, fn))) or path.islink(
                            pjoin(files_path, fn)):
                        if force in [0, 1]:
                            answer = Cmd.timed_input(question=
                                                     """The IOTest %s does not create file '%s'. Fix it! [type 'enter'] >"""\
                                    %(test_name,fn),default="y")
                        modifications['missing'].append(
                            "%s/%s/%s" %
                            (folder_name, test_name, path.basename(fname)))
                        if verbose:                            print("    > [ %s ] "%(colored%(31,"MISSING"))+\
                  "%s/%s/%s"%(folder_name,test_name,path.basename(fname)))
                    else:
                        if fname.startswith('-'):
                            veto_rules.append(fn)
                        else:
                            filesToCheck.append(fn)

            # Apply the trimming of the veto rules
            filesToCheck = [f for f in filesToCheck if f not in veto_rules]

            if update:
                # Remove files which are no longer used for comparison
                activeFiles = [self.toFileName(f) for f in filesToCheck]
                for file in glob.glob(pjoin(_hc_comparison_files,folder_name,\
                                                                test_name,'*')):
                    # Ignore the .BackUp files and directories. Also ignore
                    # a file which was previously flagged missing because it
                    # was explicitly specified in the list of files that the
                    # test *must* provide.
                    if path.basename(file).endswith('.BackUp') or \
                       path.isdir(file) or \
                       pjoin(folder_name,test_name,path.basename(file)) in \
                                                       modifications['missing']:
                        continue
                    if path.basename(file) not in activeFiles:
                        if force==0 or (force==1 and \
                         path.basename(file) not in list(reviewed_file_names.keys())):
                            answer = Cmd.timed_input(question=
                                                     """Obsolete ref. file %s in %s/%s detected, delete it? [y/n] >"""\
                                    %(path.basename(file),folder_name,test_name)
                                                                   ,default="y")
                            reviewed_file_names[path.basename(file)] = answer
                        elif (force==1 and \
                             path.basename(file) in list(reviewed_file_names.keys())):
                            answer = reviewed_file_names[path.basename(file)]
                        else:
                            answer = 'Y'

                        if answer not in ['Y', 'y', '']:
                            if verbose:
                                print("    > [ %s ] "%(colored%(31,"IGNORED"))+\
                          "file deletion %s/%s/%s"%(folder_name,test_name,
                                                            path.basename(file)))
                            continue

                        os.remove(file)
                        if verbose:                            print("    > [ %s ] "%(colored%(31,"REMOVED"))+\
                  "%s/%s/%s"%(folder_name,test_name,path.basename(file)))
                        modifications['removed'].append('/'.join(
                            str(file).split('/')[-3:]))

            # Make sure it is not filtered out by the user-filter
            if self.filesChecked_filter != ['ALL']:
                new_filesToCheck = []
                for file in filesToCheck:
                    # Try if it matches any filter
                    for filter in self.filesChecked_filter:
                        # A regular expression
                        if filter.endswith(']'):
                            split = filter[:-1].split('[')
                            # folder without the final /
                            folder = split[0][:-1]
                            if folder != path.dirname(pjoin(file)):
                                continue
                            search = re.compile('['.join(split[1:]))
                            if not search.match(path.basename(file)) is None:
                                new_filesToCheck.append(file)
                                break
                        # Just the exact filename
                        elif filter == file:
                            new_filesToCheck.append(file)
                            break
                filesToCheck = new_filesToCheck

            # Now we can scan them and process them one at a time
            # Keep track of the folders and testNames the user did not want to
            # create
            refused_Folders = []
            refused_testNames = []
            for fname in filesToCheck:
                file_path = path.abspath(pjoin(files_path, fname))
                self.assertTrue(path.isfile(file_path),
                                'File %s not found.' % str(file_path))
                comparison_path = pjoin(_hc_comparison_files,\
                                    folder_name,test_name,self.toFileName(fname))
                if not update:
                    if not os.path.isfile(comparison_path):
                        iotest.clean_output()
                        if not verbose:
                            raise MadGraph5Error("Missing ref. files for test %s\n"%test_name+\
                                "Create them with './test_manager.py -U %s'"%test_name)
                            continue
                        else:
                            print(colored % (31, 'The ref. file %s' % str(
                                '/'.join(comparison_path.split('/')[-3:])) +
                                             ' does not exist.'))
                            print(colored %
                                  (34, 'Consider creating it with ' +
                                   './test_manager.py -U %s' % test_name))
                            exit(0)
                    goal = open(comparison_path).read() % misc.get_pkg_info()
                    if not verbose:
                        self.assertFileContains(open(file_path), goal)
                    else:
                        try:
                            self.assertFileContains(open(file_path), goal)
                        except AssertionError:
                            if verbose:
                                print("    > %s differs from the reference." %
                                      fname)

                else:
                    if not path.isdir(pjoin(_hc_comparison_files,
                                            folder_name)):
                        if force == 0:
                            if folder_name in refused_Folders:
                                continue
                            answer = Cmd.timed_input(
                                question=
                                """New folder %s detected, create it? [y/n] >"""
                                % folder_name,
                                default="y")
                            if answer not in ['Y', 'y', '']:
                                refused_Folders.append(folder_name)
                                if verbose:                                    print("    > [ %s ] folder %s"\
                                   %(colored%(31,"IGNORED"),folder_name))
                                continue
                        if verbose:                            print("    > [ %s ] folder %s"%\
                                    (colored%(32,"CREATED"),folder_name))
                        os.makedirs(pjoin(_hc_comparison_files, folder_name))
                    if not path.isdir(
                            pjoin(_hc_comparison_files, folder_name,
                                  test_name)):
                        if force == 0:
                            if (folder_name, test_name) in refused_testNames:
                                continue
                            answer = Cmd.timed_input(
                                question=
                                """New test %s/%s detected, create it? [y/n] >"""
                                % (folder_name, test_name),
                                default="y")
                            if answer not in ['Y', 'y', '']:
                                refused_testNames.append(
                                    (folder_name, test_name))
                                if verbose:                                    print("    > [ %s ] test %s/%s"\
                         %(colored%(31,"IGNORED"),folder_name,test_name))
                                continue
                        if verbose:                            print("    > [ %s ] test %s/%s"\
                         %(colored%(32,"CREATED"),folder_name,test_name))
                        os.makedirs(
                            pjoin(_hc_comparison_files, folder_name,
                                  test_name))
                    # Transform the package information to make it a template
                    file = open(file_path, 'r')
                    target = file.read()
                    # So that if % appear, we cast them to %% which are not formatted.
                    target = target.replace('%', '%%')
                    # So that the version and date is automatically updated
                    target = target.replace('MadGraph5_aMC@NLO v. %(version)s, %(date)s'\
                                                           %misc.get_pkg_info(),
                                          'MadGraph5_aMC@NLO v. %(version)s, %(date)s')
                    target = target.replace('v%(version)s (%(date)s)'\
                                                           %misc.get_pkg_info(),
                                                      'v%(version)s (%(date)s)')
                    file.close()
                    if os.path.isfile(comparison_path):
                        file = open(comparison_path, 'r')
                        existing = file.read()
                        file.close()
                        if existing == target:
                            continue
                        else:
                            # Copying the existing reference as a backup
                            tmp_path = pjoin(_hc_comparison_files,folder_name,\
                                        test_name,self.toFileName(fname)+'.BackUp')
                            if os.path.isfile(tmp_path):
                                os.remove(tmp_path)
                            file = open(tmp_path, 'w')
                            file.write(target)
                            file.close()
                            if force==0 or (force==1 and path.basename(\
                            comparison_path) not in list(reviewed_file_names.keys())):
                                text = \
"""File %s in test %s/%s differs by the following (reference file first):
"""%(fname,folder_name,test_name)
                                text += misc.Popen(['diff',str(comparison_path),
                                  str(tmp_path)],stdout=subprocess.PIPE).\
                                                                communicate()[0].decode('utf-8')
                                # Remove the last newline
                                if text[-1] == '\n':
                                    text = text[:-1]
                                if (len(text.split('\n')) < 15):
                                    print(text)
                                else:
                                    pydoc.pager(text)
                                    print("Difference displayed in editor.")
                                answer = ''
                                while answer not in ['y', 'n']:
                                    answer = Cmd.timed_input(
                                        question=
                                        """Ref. file %s differs from the new one (see diff. before), update it? [y/n/h/r] >"""
                                        % fname,
                                        default="y")
                                    if answer not in ['y', 'n']:
                                        if answer == 'r':
                                            pydoc.pager(text)
                                        else:
                                            print("reference path: %s" %
                                                  comparison_path)
                                            print("code returns: %s" %
                                                  tmp_path)

                                os.remove(tmp_path)
                                reviewed_file_names[path.basename(\
                                                      comparison_path)] = answer
                            elif (force==1 and path.basename(\
                                comparison_path) in list(reviewed_file_names.keys())):
                                answer = reviewed_file_names[path.basename(\
                                                               comparison_path)]
                            else:
                                answer = 'Y'
                            if answer not in ['Y', 'y', '']:
                                if verbose:                                    print("    > [ %s ] %s"%\
                                          (colored%(31,"IGNORED"),fname))
                                continue

                            # Copying the existing reference as a backup
                            back_up_path = pjoin(_hc_comparison_files,folder_name,\
                                         test_name,self.toFileName(fname)+'.BackUp')
                            if os.path.isfile(back_up_path):
                                os.remove(back_up_path)
                            cp(comparison_path, back_up_path)
                            if verbose:                                print("    > [ %s ] %s"\
                                         %(colored%(32,"UPDATED"),fname))
                            modifications['updated'].append('/'.join(
                                comparison_path.split('/')[-3:]))
                    else:
                        if force==0 or (force==1 and path.basename(\
                            comparison_path) not in list(reviewed_file_names.keys())):
                            answer = Cmd.timed_input(
                                question=
                                """New file %s detected, create it? [y/n] >"""
                                % fname,
                                default="y")
                            reviewed_file_names[path.basename(\
                                                      comparison_path)] = answer
                        elif (force==1 and path.basename(\
                                comparison_path) in list(reviewed_file_names.keys())):
                            answer = reviewed_file_names[\
                                                 path.basename(comparison_path)]
                        else:
                            answer = 'Y'
                        if answer not in ['Y', 'y', '']:
                            if verbose:                                print("    > [ %s ] %s"%\
                                          (colored%(31,"IGNORED"),fname))
                            continue
                        if verbose:                            print("    > [ %s ] %s"%\
                                          (colored%(32,"CREATED"),fname))
                        modifications['created'].append('/'.join(
                            comparison_path.split('/')[-3:]))
                    file = open(comparison_path, 'w')
                    file.write(target)
                    file.close()

            # Clean the iotest output
            iotest.clean_output()

        # Monitor the modifications when in creation files mode by returning the
        # modifications dictionary.
        if update:
            return modifications
        else:
            return 'test_over'
예제 #7
0
    def submit(self,
               prog,
               argument=[],
               cwd=None,
               stdout=None,
               stderr=None,
               log=None):
        """Submit a job prog to an SGE cluster"""

        me_dir = os.path.realpath(os.path.join(cwd, prog)).rsplit(
            '/SubProcesses', 1)[0]
        me_dir = hashlib.md5(me_dir).hexdigest()[-10:]
        if not me_dir[0].isalpha():
            me_dir = 'a' + me_dir[1:]

        text = ""
        if cwd is None:
            #cwd = os.getcwd()
            cwd = self.def_get_path(os.getcwd())
        else:
            text = " cd %s;" % cwd
        cwd1 = self.def_get_path(cwd)
        text = " cd %s;" % cwd1
        if stdout is None:
            stdout = '/dev/null'
        else:
            stdout = self.def_get_path(stdout)
        if stderr is None:
            stderr = '/dev/null'
        elif stderr == -2:  # -2 is subprocess.STDOUT
            stderr = stdout
        else:
            stderr = self.def_get_path(stderr)

        if log is None:
            log = '/dev/null'
        else:
            log = self.def_get_path(log)

        text += prog
        if argument:
            text += ' ' + ' '.join(argument)

        #if anything slips through argument
        #print "!=== inteded change ",text.replace('/srv/nfs','')
        #text = text.replace('/srv/nfs','')
        homePath = os.getenv("HOME")
        if homePath:
            text = text.replace(homePath, '$HOME')

        logger.debug("!=== input  %s" % text)
        logger.debug("!=== output %s" % stdout)
        logger.debug("!=== error  %s" % stderr)
        logger.debug("!=== logs   %s" % log)

        command = ['qsub', '-o', stdout, '-N', me_dir, '-e', stderr, '-V']

        if self.cluster_queue and self.cluster_queue != 'None':
            command.extend(['-q', self.cluster_queue])

        a = misc.Popen(command,
                       stdout=subprocess.PIPE,
                       stderr=subprocess.STDOUT,
                       stdin=subprocess.PIPE,
                       cwd=cwd)

        output = a.communicate(text)[0]
        id = output.split(' ')[2]
        if not id.isdigit():
            raise ClusterManagmentError, 'fail to submit to the cluster: \n%s' \
                                                                        % output
        self.submitted += 1
        self.submitted_ids.append(id)
        logger.debug(output)

        return id
예제 #8
0
    def submit(self,
               prog,
               argument=[],
               cwd=None,
               stdout=None,
               stderr=None,
               log=None):
        """Submit a job prog to a Condor cluster"""

        text = """Executable = %(prog)s
                  output = %(stdout)s
                  error = %(stderr)s
                  log = %(log)s
                  %(argument)s
                  environment = CONDOR_ID=$(Cluster).$(Process)
                  Universe = vanilla
                  notification = Error
                  Initialdir = %(cwd)s
                  %(requirement)s
                  getenv=True
                  queue 1
               """

        if self.cluster_queue not in ['None', None]:
            requirement = 'Requirements = %s=?=True' % self.cluster_queue
        else:
            requirement = ''

        if cwd is None:
            cwd = os.getcwd()
        if stdout is None:
            stdout = '/dev/null'
        if stderr is None:
            stderr = '/dev/null'
        if log is None:
            log = '/dev/null'
        if not os.path.exists(prog):
            prog = os.path.join(cwd, prog)
        if argument:
            argument = 'Arguments = %s' % ' '.join(argument)
        else:
            argument = ''

        dico = {
            'prog': prog,
            'cwd': cwd,
            'stdout': stdout,
            'stderr': stderr,
            'log': log,
            'argument': argument,
            'requirement': requirement
        }

        open('submit_condor', 'w').write(text % dico)
        a = misc.Popen(['condor_submit', 'submit_condor'],
                       stdout=subprocess.PIPE)
        output = a.stdout.read()
        #Submitting job(s).
        #Logging submit event(s).
        #1 job(s) submitted to cluster 2253622.
        pat = re.compile("submitted to cluster (\d*)", re.MULTILINE)
        try:
            id = pat.search(output).groups()[0]
        except:
            raise ClusterManagmentError, 'fail to submit to the cluster: \n%s' \
                                                                        % output
        self.submitted += 1
        self.submitted_ids.append(id)
        return id