def get(self, name, start=False, cls=Block):
full_name = name
#print('get', full_name)
name, num = self._parse_name(name)
if name in self:
if start:
assert num and num > 1
block = self[name]
if num:
if start:
if num < 2:
raise KeyError('{0} is never an existing item'.format(full_name))
if num - 1 != len(block):
print(block)
raise KeyError('{0} while len is {1}'.format(full_name, len(block)))
# foo.<id> should only be used sequentially
assert len(block) == num - 1
if block.name != self.last.name:
raise ParseError('unexpected block {0} after {1}'.format(
block.name, self.last.name))
assert block == self.last # this is the same as above
return block
else:
if num:
assert num == 1
new_block = cls(name)
self[name] = new_block
return new_block
def append_line(self, line):
#print('appending', self, line)
if self.finished():
raise ParseError('block {0} is finished'.format(self))
if remove_ansi_color(line) == '':
self.elements.append(BlankLine())
else:
self.elements[0].append_line(line)
def set_parameters(self, parameters):
if not self.allow_empty:
if not self._lines:
raise ParseError('{0} is empty at end of parse'.format(self))
parameters = dict(
parameter.split('=') for parameter in parameters.split(','))
for key, value in parameters.items():
setattr(self, key, int(value))
def append_line(self, line):
if self.finished():
raise ParseError('Unexpectedly adding {0} to {1}'.format(line, self))
#print('adding {0} to {1}'.format(line, self))
nocolor_line = remove_ansi_color(line)
if nocolor_line.startswith('$ '):
if self.elements and self._single_line_response:
last_command = self.elements[-1]
assert len(last_command.lines) == 2
command = UntimedCommand()
command.append_line(line)
self.elements.append(command)
else:
if not len(self):
raise ParseError('Unexpected version line: {0}'.format(line))
last_command = self.elements[-1]
if self._single_line_response:
assert len(last_command.lines) == 1
last_command.append_line(line)
def append_line(self, line):
if self.elements:
last_command = self.elements[-1]
#print('append to block', last_command, type(last_command), line, last_command.finished())
if isinstance(last_command, TimedCommand):
raise ParseError('not expecting {0}.append_line({1})'.format(self, line))
nocolor_line = remove_ansi_color(line)
if nocolor_line.startswith('$ '):
new_command = UntimedCommand()
new_command.append_line(line)
elif self.elements:
last_command = self.elements[-1]
last_command.append_line(line)
else:
new_item = Note()
new_item.append_line(line)
def append_line(self, line):
#print('CommandBlock.append_line ', line)
nocolor_line = remove_ansi_color(line)
assert self.commands
last_command = self.commands[-1]
if isinstance(last_command, BlankLine):
last_command = self.commands[-2]
if not isinstance(last_command, Command):
if last_command and last_command.identifier in ['_unsolved_exit_code-1', '_unsolved_exit_code-2']:
pass
else:
raise ParseError('last command is {0}: {1} when inserting: {2}'.format(type(last_command), last_command, line))
if len(last_command.lines):
# clean the last command line, and remove the '$ '
exit_code_pattern = 'The command "{0}" exited with '.format(remove_ansi_color(last_command.lines[0])[2:])
if nocolor_line.startswith(exit_code_pattern):
exit_code = nocolor_line[len(exit_code_pattern):-1]
last_command.exit_code = int(exit_code)
return
elif nocolor_line and exit_code_pattern.startswith(nocolor_line):
# TODO: The exit_code_pattern needs to be a multi-line match
# e.g. happy5214/pywikibot-core/6.10
current_command = Note('_unsolved_exit_code-1')
self.commands.append(current_command)
return
exit_code_pattern = 'The command "{0}" failed and exited with '.format(remove_ansi_color(last_command.lines[0])[2:])
if nocolor_line.startswith(exit_code_pattern):
exit_code = nocolor_line[len(exit_code_pattern):].split(' during ')[0]
last_command.exit_code = int(exit_code)
return
last_command.append_line(line)
def append_line(self, line):
if remove_ansi_color(line) != '':
raise ParseError('BlankLineBlock not expecting {0}'.format(line))
self.elements.append(BlankLine())
def append_line(self, line):
assert len(self.elements) == 1
if len(self.elements[0].lines) != 0:
raise ParseError('cant insert line into {0}: {1}'.format(self, line))
assert len(self.elements[0].lines) == 0
self.elements[0].append_line(line)
def _parse(self):
if not self.body:
return {}
no_system_info = False
if 'travis_fold:start:system_info' not in self.body[:400]:
if len(self.body) < 400:
lines = self.body.strip().splitlines()
# remove blank lines
lines = [line for line in lines if line]
if lines[0].startswith('Using worker: '):
lines = lines[1:]
if len(lines) == 1 and lines[0] == 'Done: Job Cancelled':
return {}
else:
if 'travis_fold:start:system_info' in self.body:
# See https://github.com/travis-ci/travis-ci/issues/4848
raise TravisLogCorrupt
elif 'travis_fold:start:git' in self.body[:400]:
no_system_info = True
else:
raise ParseError('header not found')
blocks = BlockDict()
lines = self.body.splitlines()
if no_system_info:
print('no_system_info!')
current_block = None
current_command = None
successful = False
for line_no, line in enumerate(lines):
nocolor_line = remove_ansi_color(line)
#print('line', current_block, current_command, nocolor_line)
if nocolor_line.startswith('travis_time:start:'):
timer_id = nocolor_line[len('travis_time:start:'):]
assert timer_id
current_block = blocks.last
#print('adding start', current_block, current_block.finished(), timer_id)
if current_block in ['_versions', '_versions-continued']:
blocks.append(ScriptBlock('script'))
elif current_block and current_block.finished():
if (current_block.name.endswith('_environment_variables') or
current_block.name == '_container_notice' or
current_block.name.startswith('git')):
if 'system_info' in blocks:
build_language = blocks['system_info'].elements[0].lines[1]
build_language = build_language[len('Build language: '):]
else:
build_language = None
if build_language == 'php':
blocks.append(PHPActivateBlock('_activate'))
else:
blocks.append(SingleCommandBlock('_activate'))
elif current_block.name in '_activate':
blocks.append(CommandBlock('_versions-timed'))
elif current_block.name in ['before_script', 'install', 'install.bundler'] or current_block.name in ['before_script-continued', 'install-continued']:
blocks.append(ScriptBlock('script'))
else:
raise ParseError('unexpected line after {0}: {1}'.format(current_block, line))
current_block = blocks.last
#print('start', current_block)
if current_block is None:
print(blocks)
raise ParseError('unexpected start: {0}'.format(line))
allow_empty = current_block.allow_empty()
current_command = TimedCommand(timer_id, allow_empty)
current_block.append(current_command)
#print('start timed inserted', current_block)
elif nocolor_line.startswith('travis_time:end:'):
data = nocolor_line[len('travis_time:end:'):]
end_timer_id, parameters = data.split(':', 1)
current_command = blocks.last.last_item
if not isinstance(current_command, TimedCommand):
if isinstance(blocks.last, JobCancelled):
last_timed_command = blocks[-2].last_item
assert isinstance(last_timed_command, TimedCommand)
current_command = ContinuedTimedCommand(last_timed_command, allow_empty=True)
else:
raise ParseError('Last item is a {0} and not a TimedCommand: {1}'.format(type(current_command), current_command))
#print('end', current_block, current_command, end_timer_id)
if current_command.identifier != end_timer_id:
raise ParseError('{0} is not {1}'.format(current_command, end_timer_id))
#if not current_command.lines:
# print('end with no lines', end_timer_id, current_block, current_command)
# sys.exit()
current_command.set_parameters(parameters)
current_command = None
elif nocolor_line.startswith('travis_fold:start:'):
block_name = nocolor_line[len('travis_fold:start:'):]
last_block = blocks.last
cls = None
if no_system_info and block_name == 'git.1':
print('odd git block')
if blocks.last == '_top_env':
cls = AutoGitBlock
else:
cls = OldGitBlock
elif block_name == 'apt':
cls = AptBlock
elif block_name == 'system_info':
cls = OneNoteBlock
elif block_name == 'announce':
cls = AutoCommandBlock
elif block_name == 'before_install':
cls = MixedCommandBlock
else:
cls = MixedCommandBlock
current_block = blocks.get(block_name, start=True, cls=cls)
# The number of git blocks is variable
# If another one appears, it is OK.
if current_block.name == 'git' and current_block._finished:
current_block._finished = False
if last_block and current_block != last_block and last_block.finished() == False and last_block.name != 'git':
raise ParseError('start of {0} during {1} unexpected after {2} ({3})'.format(block_name, current_block, last_block, last_block.finished()))
if no_system_info and block_name == 'git.3' and current_block.__class__ == OldGitBlock:
assert len(blocks) == 2 # block 0 should be _worker
assert len(current_block) > 1 # TODO: better assert
current_command = UntimedCommand('_untimed_git_checkout')
current_block.commands.append(current_command)
current_block._finished = None
elif nocolor_line.startswith('travis_fold:end:'):
block_name = nocolor_line[len('travis_fold:end:'):]
current_block = blocks[block_name]
if not current_block == blocks.last:
if blocks.last.name == current_block.name + '-continued':
blocks.last._finished = True
elif isinstance(blocks.last, JobCancelled): # not used
current_block = blocks[-2]
last_timed_command = blocks[-2].last_item
assert isinstance(last_timed_command, TimedCommand)
else:
raise ParseError('{0} != {1}'.format(current_block, blocks.last))
if hasattr(current_block.last_item, '_finished'):
current_block.last_item._finished = True
current_block._finished = True
current_block = None
current_command = None
if block_name == 'announce': # wikimedia/wikipedia-ios/543.1 needs this
blocks.append(AutoVersionCommandBlock('_versions-extra'))
else:
# ruby coveralls includes travis variables in its payload
if 'travis_' in line and not '"travis_' in line:
raise ParseError(
'unexpected travis_ in {0} while parsing {1}'.format(
line, current_block))
new_block = find_new_block(line)
if new_block is not None:
#print('found new block', new_block)
#if blocks:
# print('last block was', blocks.last)
blocks.append(new_block)
# Single line item?
if new_block.finished():
current_block = None
current_block = new_block
if current_block.elements:
current_command = current_block.elements[-1]
continue
else:
current_command = None
#print(line)
if no_system_info and blocks.last and blocks.last.name == '_worker' and blocks.last.finished():
# probably a cpp block, as most environments have a 'system_info' block
# which is handled above
blocks.append(AutoCommandBlock('_top_env'))
if blocks and blocks.last.name in ['rvm'] and blocks.last.finished():
if 'jupiter' in blocks['_worker'].elements[0].lines[0]:
print('using OSXRubyVersionBlock')
current_block = OSXRubyVersionBlock('_versions-odd')
else:
current_block = AutoVersionCommandBlock('_versions')
blocks.append(current_block)
elif current_command is None and current_block in ['git.checkout', 'git.submodule'] and nocolor_line.startswith('The command "git ') and '" failed and exited with ' in nocolor_line:
# TODO: match the command in the quotes
current_command = current_block.commands[-1]
if current_block == '_activate' and current_block.finished():
print('after _activate')
blocks.append(AutoVersionCommandBlock('_versions'))
current_block = blocks.last
if current_block == '_job_cancelled' and current_block.finished():
if isinstance(blocks[-2].elements[-1], TimedCommand):
current_block = blocks[-2].__class__(blocks[-2].name + '-continued')
current_command = ContinuedTimedCommand(blocks[-2].elements[-1], allow_empty=True)
current_block.append(current_command)
else:
current_block = Block('_stuff_after_job_cancelled-' + str(line_no))
blocks.append(current_block)
current_block = blocks.last
if current_block is not None:
#print(current_block.name, current_block.finished(), line)
if no_system_info and current_block == 'before_install' and current_block.finished():
blocks.append(AutoScriptBlock('script'))
current_block = blocks.last
if current_block.finished():
print('current block is finished', current_block)
if nocolor_line.startswith('$ ') and nocolor_line.endswith('-- version'):
blocks.append(AutoVersionCommandBlock('_versions'))
current_block = blocks.last
elif nocolor_line == '':
blocks.append(BlankLineBlock('_unexpected_blank_lines-' + str(line_no)))
continue
if 'Done' in line:
print('adding {0} to block'.format(line))
try:
current_block.append_line(line)
except Exception:
print('failed on line {0}: {1}'.format(line_no, line))
raise
elif nocolor_line: # ignore blank lines
previous_block_name = None if not blocks else blocks[-1].name
raise ParseError('unexpected line after {0}: {1!r}'.format(previous_block_name, line))
for block in blocks:
if block.__class__ == Block:
raise ParseError('Block needs to be converted to something else')
if blocks.last == '_done':
done_block = blocks.last
assert done_block.exit_code is not None
previous_block = blocks[-2]
if previous_block.name.startswith('_unexpected_blank_lines'):
previous_block = blocks[-3]
if previous_block.name == 'after_success': # wikimedia/wikipedia-ios/543.1
previous_block = blocks[-4]
assert previous_block.name in ['script', 'script-continued']
last_command = previous_block.commands[-1]
assert isinstance(last_command, Command)
if last_command.exit_code is None:
pass
#raise ParseError('Build exit with {0}, but last command {1} doesnt have an exit code'.format(done_block.exit_code, last_command))
elif done_block.exit_code > 0 and last_command.exit_code == 0:
print('Build exit with {0}, but last command exit code was {1}'.format(done_block.exit_code, last_command.exit_code))
elif done_block.exit_code > 0 and last_command.exit_code == 0:
raise ParseError('Build exit with {0}, but last command exit code was {1}'.format(done_block.exit_code, last_command.exit_code))
continue
return blocks
def append(self, block):
if block.name in self:
raise ParseError('{0} already in {1}'.format(block, self))
self[block.name] = block