def test_multiple_targets(self):
target1 = Target.from_simple_slug('foo/bar')
target2 = Target.from_simple_slug('foo/baz')
_travis = _get_travispy()
jobs = get_jobs(_travis, [target1, target2])
assert len(jobs) == 0
def test_multiple_targets(self):
target1 = Target.from_simple_slug('foo/bar')
target2 = Target.from_simple_slug('foo/baz')
_travis = _get_travispy()
jobs = get_jobs(_travis, [target1, target2])
assert len(jobs) == 0
def test_multiple(self):
target1 = Target.from_simple_slug('travispy/on_pypy')
target2 = Target.from_simple_slug('travispy/on_py34')
_travis = _get_travispy()
jobs = get_jobs(_travis, [target1, target2])
assert len(jobs) == 2
assert jobs[0].repository_id == 2598880
assert jobs[1].repository_id == 2598879
def test_multiple(self):
target1 = Target.from_simple_slug('travispy/on_pypy')
target2 = Target.from_simple_slug('travispy/on_py34')
_travis = _get_travispy()
jobs = get_jobs(_travis, [target1, target2])
assert len(jobs) == 2
assert jobs[0].repository_id == 2598880
assert jobs[1].repository_id == 2598879
def test_empty(self):
target = Target()
assert target.user is None
assert target.project is None
assert target.build_id is None
assert target.job_id is None
assert target.build_number is None
assert target.job_number is None
def test_build_url(self):
target = Target.from_url('//travis-ci.org/foo/bar/builds/9999')
assert target.user == 'foo'
assert target.project == 'bar'
assert target.slug == 'foo/bar'
assert target.build_id == 9999
assert target.job_id is None
assert target.build_number is None
assert target.job_number is None
def test_job_url(self):
target = Target.from_url('//travis-ci.org/foo/bar/jobs/10000')
assert target.user == 'foo'
assert target.project == 'bar'
assert target.slug == 'foo/bar'
assert target.build_id is None
assert target.job_id == 10000
assert target.build_number is None
assert target.job_number is None
def test_build_url(self):
target = Target.from_url('//travis-ci.org/foo/bar/builds/9999')
assert target.user == 'foo'
assert target.project == 'bar'
assert target.slug == 'foo/bar'
assert target.build_id == 9999
assert target.job_id is None
assert target.build_number is None
assert target.job_number is None
def test_single(self):
target = Target.from_simple_slug('travispy/on_pypy')
_travis = _get_travispy()
jobs = get_jobs(_travis, target)
assert len(jobs) == 1
job = jobs[0]
assert job.repository_id == 2598880
def test_logical_multiple_job_build(self):
target = Target.from_extended_slug('menegazzo/travispy#101.3')
_travis = _get_travispy()
job = get_historical_job(_travis, target)
assert job.repository_id == 2419489
assert job.number == '101.3'
assert job.id == 82131391
def test_logical_single_job_build(self):
target = Target.from_extended_slug('travispy/on_pypy#1.1')
_travis = _get_travispy()
job = get_historical_job(_travis, target)
assert job.repository_id == 2598880
assert job.number == '1.1'
assert job.id == 28881356
def test_hash(self):
target = Target.from_extended_slug('foo/bar#10.1')
assert target.user == 'foo'
assert target.project == 'bar'
assert target.slug == 'foo/bar'
assert target.build_id is None
assert target.job_id is None
assert target.build_number == 10
assert target.job_number == 1
def test_logical_single_job_build(self):
target = Target.from_extended_slug('travispy/on_pypy#1.1')
_travis = _get_travispy()
job = get_historical_job(_travis, target)
assert job.repository_id == 2598880
assert job.number == '1.1'
assert job.id == 28881356
def test_colon(self):
target = Target.from_extended_slug('foo/bar:10000')
assert target.user == 'foo'
assert target.project == 'bar'
assert target.slug == 'foo/bar'
assert target.build_id is None
assert target.job_id == 10000
assert target.build_number is None
assert target.job_number is None
def test_no_protocol_project_url(self):
target = Target.from_url('//travis-ci.org/foo/bar')
assert target.user == 'foo'
assert target.project == 'bar'
assert target.slug == 'foo/bar'
assert target.build_id is None
assert target.job_id is None
assert target.build_number is None
assert target.job_number is None
def test_colon(self):
target = Target.from_extended_slug('foo/bar:10000')
assert target.user == 'foo'
assert target.project == 'bar'
assert target.slug == 'foo/bar'
assert target.build_id is None
assert target.job_id == 10000
assert target.build_number is None
assert target.job_number is None
def test_at(self):
target = Target.from_extended_slug('foo/bar@9999')
assert target.user == 'foo'
assert target.project == 'bar'
assert target.slug == 'foo/bar'
assert target.build_id == 9999
assert target.job_id is None
assert target.build_number is None
assert target.job_number is None
def test_hash(self):
target = Target.from_extended_slug('foo/bar#10.1')
assert target.user == 'foo'
assert target.project == 'bar'
assert target.slug == 'foo/bar'
assert target.build_id is None
assert target.job_id is None
assert target.build_number == 10
assert target.job_number == 1
def test_job_url(self):
target = Target.from_url('//travis-ci.org/foo/bar/jobs/10000')
assert target.user == 'foo'
assert target.project == 'bar'
assert target.slug == 'foo/bar'
assert target.build_id is None
assert target.job_id == 10000
assert target.build_number is None
assert target.job_number is None
def test_at(self):
target = Target.from_extended_slug('foo/bar@9999')
assert target.user == 'foo'
assert target.project == 'bar'
assert target.slug == 'foo/bar'
assert target.build_id == 9999
assert target.job_id is None
assert target.build_number is None
assert target.job_number is None
def test_basic_slug(self):
target = Target.from_simple_slug('foo/bar')
assert target.user == 'foo'
assert target.project == 'bar'
assert target.slug == 'foo/bar'
assert target.build_id is None
assert target.job_id is None
assert target.build_number is None
assert target.job_number is None
def test_http_project_url(self):
target = Target.from_url('http://travis-ci.org/foo/bar')
assert target.user == 'foo'
assert target.project == 'bar'
assert target.slug == 'foo/bar'
assert target.build_id is None
assert target.job_id is None
assert target.build_number is None
assert target.job_number is None
def test_single(self):
target = Target.from_simple_slug('travispy/on_pypy')
_travis = _get_travispy()
jobs = get_jobs(_travis, target)
assert len(jobs) == 1
job = jobs[0]
assert job.repository_id == 2598880
def test_logical_multiple_job_build(self):
target = Target.from_extended_slug('menegazzo/travispy#101.3')
_travis = _get_travispy()
job = get_historical_job(_travis, target)
assert job.repository_id == 2419489
assert job.number == '101.3'
assert job.id == 82131391
def test_basic_slug(self):
target = Target.from_simple_slug('foo/bar')
assert target.user == 'foo'
assert target.project == 'bar'
assert target.slug == 'foo/bar'
assert target.build_id is None
assert target.job_id is None
assert target.build_number is None
assert target.job_number is None
def _select_repo_builds(targets, slug):
"""Filter targets for one repo."""
targets = [target for target in targets if target.slug == slug]
build_numbers = set(target.build_number for target in targets
if isinstance(target, Target))
builds = [
Target(slug=slug, build_number=build_number)
for build_number in build_numbers
]
return builds
def get_stored_targets(base_dir, log_filename_format=None):
"""Get parsed stored targets."""
assert log_filename_format
if '{job.state}' in log_filename_format:
log_filename_format = log_filename_format.replace(
'{job.state}', '{job.state:w}')
if '{job.number}' in log_filename_format:
log_filename_format = log_filename_format.replace(
'{job.number}', '{job.number:f}')
parser = parse.compile(log_filename_format)
if base_dir[-1] != '/':
base_dir = base_dir + '/'
filenames = get_files(base_dir)
targets = []
for filename in filenames:
parsed_filename = parser.parse(filename)
if not parsed_filename:
__logs__.warning('Unexpected filename {0}'.format(filename))
continue
target = Target()
try:
target.slug = parsed_filename['job.repository.slug']
except KeyError:
pass
try:
target.number = str(parsed_filename['job.number'])
except KeyError:
pass
try:
target.state = parsed_filename['job.state']
except KeyError:
pass
targets.append(target)
return targets
def get_stored_targets(base_dir, log_filename_format=None):
"""Get parsed stored targets."""
assert log_filename_format
if '{job.state}' in log_filename_format:
log_filename_format = log_filename_format.replace(
'{job.state}', '{job.state:w}')
if '{job.number}' in log_filename_format:
log_filename_format = log_filename_format.replace(
'{job.number}', '{job.number:f}')
parser = parse.compile(log_filename_format)
if base_dir[-1] != '/':
base_dir = base_dir + '/'
filenames = get_files(base_dir)
targets = []
for filename in filenames:
parsed_filename = parser.parse(filename)
if not parsed_filename:
__logs__.warning('Unexpected filename {0}'.format(filename))
continue
target = Target()
try:
target.slug = parsed_filename['job.repository.slug']
except KeyError:
pass
try:
target.number = str(parsed_filename['job.number'])
except KeyError:
pass
try:
target.state = parsed_filename['job.state']
except KeyError:
pass
targets.append(target)
return targets
def skip_stored(targets, base_dir, log_filename_format=None):
"""Optimistically skip targets that have been fetched."""
assert log_filename_format
stored_targets = get_stored_targets(base_dir, log_filename_format)
new_targets = []
for target in targets:
if not isinstance(target, Target):
target = Target._from_travispy_obj(target)
slug = target.slug
stored_repo_targets = _select_repo_builds(stored_targets, slug)
if target not in stored_repo_targets:
__logs__.debug('target {0} not found in {1!r}'.format(
target, stored_repo_targets))
new_targets.append(target)
else:
__logs__.info('skipping existing {0}'.format(target))
return new_targets
def skip_stored(targets, base_dir, log_filename_format=None):
"""Optimistically skip targets that have been fetched."""
assert log_filename_format
stored_targets = get_stored_targets(base_dir, log_filename_format)
new_targets = []
for target in targets:
if not isinstance(target, Target):
target = Target._from_travispy_obj(target)
slug = target.slug
stored_repo_targets = _select_repo_builds(stored_targets, slug)
if target not in stored_repo_targets:
__logs__.debug('target {0} not found in {1!r}'.format(
target, stored_repo_targets))
new_targets.append(target)
else:
__logs__.info('skipping existing {0}'.format(target))
return new_targets
def main():
"""Main handler."""
options = config.get_options()
if not options or options.verbose:
logging.basicConfig(level=logging.DEBUG,
format='%(asctime)s %(levelname)9s %(message)s',
datefmt='%Y-%m-%d %H:%M:%S')
__logs__.debug('{0!r}'.format(options))
t = config._get_travispy()
gh = config._get_github()
if t:
user = t.user()
else:
user = None
targets = []
for target_string in options.targets:
if '://' in target_string:
identifier = Target.from_url(target_string)
targets.append(identifier)
else:
identifier = Target.from_extended_slug(target_string)
targets.append(identifier)
if options.refresh:
slugs = get_stored_repo_slugs(options.dir, options.format)
for slug in slugs:
identifier = Target.from_simple_slug(slug)
targets.append(identifier)
if options.self:
assert user
targets += get_user_repos(t, user)
if options.forks:
for target in targets:
slugs = get_forks(gh, target.slug)
forks = get_travis_repos(t, slugs)
targets += forks
if options.all or options.old:
count = None if options.all else options.count
new_targets = []
for target in targets:
new_targets += list(
islice(get_historical_builds(t, target.slug, _load_jobs=False),
count))
targets = new_targets
# TODO: dont enumerate jobs if the files are all dated after the build end
# TODO: enumerate all files starting with the job number,
# and delete -started, -etc, when state is 'passed.
if not options.force:
targets = skip_stored(targets, options.dir, options.format)
jobs = get_jobs(t, targets)
if options.wait:
jobs = get_completed(t, jobs, options.sleep)
for job in jobs:
download_job_log(options.dir, job, options.format)
def main():
"""Main handler."""
options = config.get_options()
if not options or options.verbose:
logging.basicConfig(
level=logging.DEBUG,
format='%(asctime)s %(levelname)9s %(message)s',
datefmt='%Y-%m-%d %H:%M:%S')
__logs__.debug('{0!r}'.format(options))
t = config._get_travispy()
gh = config._get_github()
if t:
user = t.user()
else:
user = None
targets = []
for target_string in options.targets:
if '://' in target_string:
identifier = Target.from_url(target_string)
targets.append(identifier)
else:
identifier = Target.from_extended_slug(target_string)
targets.append(identifier)
if options.refresh:
slugs = get_stored_repo_slugs(options.dir, options.format)
for slug in slugs:
identifier = Target.from_simple_slug(slug)
targets.append(identifier)
if options.self:
assert user
targets += get_user_repos(t, user)
if options.forks:
for target in targets:
slugs = get_forks(gh, target.slug)
forks = get_travis_repos(t, slugs)
targets += forks
if options.all or options.old:
count = None if options.all else options.count
new_targets = []
for target in targets:
new_targets += list(islice(
get_historical_builds(t, target.slug, _load_jobs=False),
count))
targets = new_targets
# TODO: dont enumerate jobs if the files are all dated after the build end
# TODO: enumerate all files starting with the job number,
# and delete -started, -etc, when state is 'passed.
if not options.force:
targets = skip_stored(targets, options.dir, options.format)
jobs = get_jobs(t, targets)
if options.wait:
jobs = get_completed(t, jobs, options.sleep)
for job in jobs:
download_job_log(options.dir, job, options.format)
def test_single_target(self):
target = Target.from_simple_slug('foo/bar')
_travis = _get_travispy()
jobs = get_jobs(_travis, target)
assert len(jobs) == 0
def test_logical_duplicate_build(self):
target = Target.from_extended_slug('menegazzo/travispy#80.3')
_travis = _get_travispy()
pytest.raises(AssertionError, get_historical_build, _travis, target)
def test_logical_duplicate_build(self):
target = Target.from_extended_slug('menegazzo/travispy#80.3')
_travis = _get_travispy()
pytest.raises(AssertionError, get_historical_build, _travis, target)
def test_single_target(self):
target = Target.from_simple_slug('foo/bar')
_travis = _get_travispy()
jobs = get_jobs(_travis, target)
assert len(jobs) == 0