Esempio n. 1
0
    def test_date_time(self):
# date 201609190454.17, i.e., 19 September 2016, 04:54:17
        valid_dts = ['0454', '0454.17', '190454', '190454.17',
                     '09190454', '09190454.17', '1609190454',
                     '1609190454.17', '201609190454', '201609190454.17']
        invalid_dts = ['454', '9190454']
        parser = PbsOptionParser(self._config, self._event_defs, None)
        for dt_str in valid_dts:
            self.assertTrue(parser.is_valid_datetime(dt_str),
                            msg="date time string '{0}'".format(dt_str))
        for dt_str in invalid_dts:
            self.assertFalse(parser.is_valid_datetime(dt_str),
                             msg="date time string '{0}'".format(dt_str))
Esempio n. 2
0
 def __init__(self, config, event_defs, pbs_directive='#PBS'):
     '''Constructor'''
     super(PbsScriptParser, self).__init__(event_defs)
     self._config = config
     self._job = PbsJob(self._config)
     self._pbs_directive = pbs_directive
     regex = r'\s*{0}\s+(.+)$'.format(pbs_directive)
     self._pbs_re = re.compile(regex)
     regex = r'\s+{0}\s+(.+)$'.format(pbs_directive)
     self._pbs_indented_re = re.compile(regex)
     regex = r'\s*{0}\s+(.+)$'.format(pbs_directive)
     self._pbs_extract_re = re.compile(regex)
     self._pbs_option_parser = PbsOptionParser(self._config, event_defs,
                                               self._job)
     self._state = None
     self._line_nr = 0
     self._pbs = []
Esempio n. 3
0
 def update_job_info(self, job):
     '''update the job info based on this event'''
     job.state = self.type
     for key, attr_name in PbsJobEvent._key_map.items():
         if self.has_info(key):
             job.__setattr__(attr_name, self.info(key))
     prefix = 'Resource_List.'
     for key in self._info:
         if key.startswith(prefix):
             resource = key[len(prefix):]
             if resource == 'nodes':
                 node_specs = PbsOptionParser.parse_node_spec_str(self.info(key))
                 job.add_resource_spec(resource, node_specs)
             elif resource == 'feature':
                 job.add_resource_spec('features', [self.info(key)])
             else:
                 job.add_resource_spec(resource, self.info(key))
     prefix = 'resources_used.'
     for key in self._info:
         if key.startswith(prefix):
             resource = key[len(prefix):]
             job.add_resource_used(resource, self.info(key))
Esempio n. 4
0
 def update_job_info(self, job):
     '''update the job info based on this event'''
     job.state = self.type
     for key, attr_name in PbsJobEvent._key_map.iteritems():
         if self.has_info(key):
             job.__setattr__(attr_name, self.info(key))
     prefix = 'Resource_List.'
     for key in self._info:
         if key.startswith(prefix):
             resource = key[len(prefix):]
             if resource == 'nodes':
                 node_specs = PbsOptionParser.parse_node_spec_str(self.info(key))
                 job.add_resource_spec(resource, node_specs)
             elif resource == 'feature':
                 job.add_resource_spec('features', [self.info(key)])
             else:
                 job.add_resource_spec(resource, self.info(key))
     prefix = 'resources_used.'
     for key in self._info:
         if key.startswith(prefix):
             resource = key[len(prefix):]
             job.add_resource_used(resource, self.info(key))
Esempio n. 5
0
class PbsScriptParser(EventLogger):
    '''Parser for PBS torque job files'''
    def __init__(self, config, event_defs, pbs_directive='#PBS'):
        '''Constructor'''
        super(PbsScriptParser, self).__init__(event_defs)
        self._config = config
        self._job = PbsJob(self._config)
        self._pbs_directive = pbs_directive
        regex = r'\s*{0}\s+(.+)$'.format(pbs_directive)
        self._pbs_re = re.compile(regex)
        regex = r'\s+{0}\s+(.+)$'.format(pbs_directive)
        self._pbs_indented_re = re.compile(regex)
        regex = r'\s*{0}\s+(.+)$'.format(pbs_directive)
        self._pbs_extract_re = re.compile(regex)
        regex = r'\s*-[A-Za-z]'
        self._pbs_option_re = re.compile(regex)
        self._pbs_option_parser = PbsOptionParser(self._config, event_defs,
                                                  self._job)
        self._state = None
        self._line_nr = 0
        self._pbs = []
        self._script_first_line_nr = None

    @property
    def script_first_line_nr(self):
        '''return the first line number of the Bash script part'''
        return self._script_first_line_nr

    def parse_file(self, pbs_file):
        '''parse a PBS file'''
        self._job.name = os.path.basename(pbs_file.name)
        self._state = 'start'
        self._line_nr = 0
        for line in pbs_file:
            self._line_nr += 1
            self.check_encoding(line)
            if self.is_comment(line):
                continue
            if len(line.strip()) == 0:
                continue
            if self._state == 'start':
                self.parse_shebang(line)
            elif self._state == 'pbs':
                self.parse_pbs(line)
            else:
                self.parse_script(line)
        if self._state == 'start':
            self.reg_event('no_script')
        elif self._state == 'pbs':
            self.reg_event('no_script')

    @property
    def job(self):
        '''returns a PbsJob object representing the job script'''
        return self._job

    def check_encoding(self, line):
        '''checks ASCII encoding and line endings'''
        try:
            line.decode('ascii')
        except UnicodeDecodeError:
            self.reg_event('non_ascii')
        if line.endswith('\r\n'):
            self.reg_event('dos_format')
        if line.endswith('\r'):
            self.reg_event('mac_format')

    def is_comment(self, line):
        '''returns True if the line is a comment'''
        return (re.match(r'\s*#', line)
                and not (self.is_shebang(line) or self.is_spaced_pbs(line)
                         or self.is_indented_pbs(line) or self.is_pbs(line)))

    def is_shebang(self, line):
        '''returns True if the line is a shebang'''
        return line.startswith('#!')

    def is_spaced_pbs(self, line):
        '''checks whether an extra space is added between hash and text
           in PBS directive, only for default directive string'''
        return self._pbs_directive == '#PBS' and re.match(
            r'\s*#\s+PBS\s+', line)

    def is_pbs(self, line):
        '''returns True if the line is a PBS directive'''
        return self._pbs_re.match(line)

    def is_indented_pbs(self, line):
        '''returns True if the line contains an indented PBS directive'''
        return self._pbs_indented_re.match(line)

    def parse_shebang(self, line):
        '''parse shebang part of PBS file'''
        if self.is_shebang(line):
            self._job.shebang = line.strip()
            if self._line_nr > 1:
                self.reg_event('misplaced_shebang')
            self._state = 'pbs'
        else:
            self.reg_event('missing_shebang')
            self._state = 'pbs'
            self.parse_pbs(line)

    def parse_pbs(self, line):
        '''parse PBS directives part of a PBS file'''
        if self.is_shebang(line):
            self.reg_event('misplaced_shebang')
        elif self.is_spaced_pbs(line):
            self.reg_event('space_in_pbs_dir')
        elif self.is_pbs(line):
            if self.is_indented_pbs(line):
                self.reg_event('indented_pbs_dir')
            match = self._pbs_extract_re.match(line)
            if match:
                option = match.group(1)
                if self._pbs_option_re.match(option):
                    self._pbs_option_parser.parse_args(option)
                    self.merge_events(self._pbs_option_parser.events)
                else:
                    self.reg_event('malformed_pbs_dir')
            else:
                self.reg_event('malformed_pbs_dir')
        else:
            self._state = 'script'
            if not self._script_first_line_nr:
                self._script_first_line_nr = self._line_nr
            self.parse_script(line)

    def parse_script(self, line):
        '''parse shell script part of a PBS file'''
        if self.is_shebang(line):
            self.reg_event('misplaced_shebang')
        if self.is_pbs(line):
            self.reg_event('misplace_pbs_dir')
        self._job.add_script_line(self._line_nr, line)
Esempio n. 6
0
class PbsScriptParser(EventLogger):
    '''Parser for PBS torque job files'''

    def __init__(self, config, event_defs, pbs_directive='#PBS'):
        '''Constructor'''
        super(PbsScriptParser, self).__init__(event_defs)
        self._config = config
        self._job = PbsJob(self._config)
        self._pbs_directive = pbs_directive
        regex = r'\s*{0}\s+(.+)$'.format(pbs_directive)
        self._pbs_re = re.compile(regex)
        regex = r'\s+{0}\s+(.+)$'.format(pbs_directive)
        self._pbs_indented_re = re.compile(regex)
        regex = r'\s*{0}\s+(.+)$'.format(pbs_directive)
        self._pbs_extract_re = re.compile(regex)
        regex = r'\s*-[A-Za-z]'
        self._pbs_option_re = re.compile(regex)
        self._pbs_option_parser = PbsOptionParser(self._config, event_defs,
                                                  self._job)
        self._state = None
        self._line_nr = 0
        self._pbs = []
        self._script_first_line_nr = None

    @property
    def script_first_line_nr(self):
        '''return the first line number of the Bash script part'''
        return self._script_first_line_nr

    def parse_file(self, pbs_file):
        '''parse a PBS file'''
        self._job.name = os.path.basename(pbs_file.name)
        self._state = 'start'
        self._line_nr = 0
        for line in pbs_file:
            self._line_nr += 1
            self.check_encoding(line)
            if self.is_comment(line):
                continue
            if len(line.strip()) == 0:
                continue
            if self._state == 'start':
                self.parse_shebang(line)
            elif self._state == 'pbs':
                self.parse_pbs(line)
            else:
                self.parse_script(line)
        if self._state == 'start':
            self.reg_event('no_script')
        elif self._state == 'pbs':
            self.reg_event('no_script')

    @property
    def job(self):
        '''returns a PbsJob object representing the job script'''
        return self._job

    def check_encoding(self, line):
        '''checks ASCII encoding and line endings'''
        try:
            line.decode('ascii')
        except UnicodeDecodeError:
            self.reg_event('non_ascii')
        if line.endswith('\r\n'):
            self.reg_event('dos_format')
        if line.endswith('\r'):
            self.reg_event('mac_format')

    def is_comment(self, line):
        '''returns True if the line is a comment'''
        return (re.match(r'\s*#', line) and
                not (self.is_shebang(line) or
                     self.is_spaced_pbs(line) or
                     self.is_indented_pbs(line) or
                     self.is_pbs(line)))

    def is_shebang(self, line):
        '''returns True if the line is a shebang'''
        return line.startswith('#!')

    def is_spaced_pbs(self, line):
        '''checks whether an extra space is added between hash and text
           in PBS directive, only for default directive string'''
        return self._pbs_directive == '#PBS' and re.match(r'\s*#\s+PBS\s+',
                                                          line)

    def is_pbs(self, line):
        '''returns True if the line is a PBS directive'''
        return self._pbs_re.match(line)

    def is_indented_pbs(self, line):
        '''returns True if the line contains an indented PBS directive'''
        return self._pbs_indented_re.match(line)

    def parse_shebang(self, line):
        '''parse shebang part of PBS file'''
        if self.is_shebang(line):
            self._job.shebang = line.strip()
            if self._line_nr > 1:
                self.reg_event('misplaced_shebang')
            self._state = 'pbs'
        else:
            self.reg_event('missing_shebang')
            self._state = 'pbs'
            self.parse_pbs(line)

    def parse_pbs(self, line):
        '''parse PBS directives part of a PBS file'''
        if self.is_shebang(line):
            self.reg_event('misplaced_shebang')
        elif self.is_spaced_pbs(line):
            self.reg_event('space_in_pbs_dir')
        elif self.is_pbs(line):
            if self.is_indented_pbs(line):
                self.reg_event('indented_pbs_dir')
            match = self._pbs_extract_re.match(line)
            if match:
                option = match.group(1)
                if self._pbs_option_re.match(option):
                    self._pbs_option_parser.parse_args(option)
                    self.merge_events(self._pbs_option_parser.events)
                else:
                    self.reg_event('malformed_pbs_dir')
            else:
                self.reg_event('malformed_pbs_dir')
        else:
            self._state = 'script'
            if not self._script_first_line_nr:
                self._script_first_line_nr = self._line_nr
            self.parse_script(line)

    def parse_script(self, line):
        '''parse shell script part of a PBS file'''
        if self.is_shebang(line):
            self.reg_event('misplaced_shebang')
        if self.is_pbs(line):
            self.reg_event('misplace_pbs_dir')
        self._job.add_script_line(self._line_nr, line)