def run(self,
release_url,
tests,
upload_build_id,
upload_release_name,
heartbeat=None):
if not upload_release_name:
upload_release_name = str(datetime.datetime.utcnow())
yield heartbeat('Creating release %s' % upload_release_name)
release_number = yield release_worker.CreateReleaseWorkflow(
upload_build_id, upload_release_name, release_url)
pending_uploads = []
for test in tests:
item = release_worker.RequestRunWorkflow(
upload_build_id,
upload_release_name,
release_number,
test.name,
url=test.run_url,
config_data=test.run_config_data,
ref_url=test.ref_url,
ref_config_data=test.ref_config_data)
pending_uploads.append(item)
yield heartbeat('Requesting %d runs' % len(pending_uploads))
yield pending_uploads
yield heartbeat('Marking runs as complete')
release_url = yield release_worker.RunsDoneWorkflow(
upload_build_id, upload_release_name, release_number)
yield heartbeat('Results viewable at: %s' % release_url)
def run(self,
release_url,
tests,
upload_build_id,
upload_release_name,
heartbeat=None):
if not upload_release_name:
upload_release_name = str(datetime.datetime.now())
yield heartbeat('Creating release %s' % upload_release_name)
release_number = yield release_worker.CreateReleaseWorkflow(
upload_build_id, upload_release_name, release_url)
pending_uploads = []
for test in tests:
item = RunTest(upload_build_id,
upload_release_name,
release_number,
test,
heartbeat=heartbeat)
pending_uploads.append(item)
yield heartbeat('Uploading %d runs' % len(pending_uploads))
yield pending_uploads
yield heartbeat('Marking runs as complete')
release_url = yield release_worker.RunsDoneWorkflow(
upload_build_id, upload_release_name, release_number)
yield heartbeat('Results viewable at: %s' % release_url)
def run(self,
new_url,
baseline_url,
upload_build_id,
upload_release_name=None,
heartbeat=None):
# TODO: Make the default release name prettier.
if not upload_release_name:
upload_release_name = str(datetime.datetime.utcnow())
yield heartbeat('Creating release %s' % upload_release_name)
release_number = yield release_worker.CreateReleaseWorkflow(
upload_build_id, upload_release_name, new_url)
config_dict = {
'viewportSize': {
'width': 1280,
'height': 1024,
}
}
if FLAGS.width:
config_dict['viewportSize']['width'] = FLAGS.width
if FLAGS.height:
config_dict['viewportSize']['height'] = FLAGS.height
if FLAGS.inject_css:
config_dict['injectCss'] = FLAGS.inject_css
if FLAGS.inject_js:
config_dict['injectJs'] = FLAGS.inject_js
if FLAGS.http_username:
config_dict['httpUserName'] = FLAGS.http_username
if FLAGS.http_password:
config_dict['httpPassword'] = FLAGS.http_password
config_data = json.dumps(config_dict)
url_parts = urlparse.urlparse(new_url)
yield heartbeat('Requesting captures')
yield release_worker.RequestRunWorkflow(upload_build_id,
upload_release_name,
release_number,
url_parts.path or '/',
new_url,
config_data,
ref_url=baseline_url,
ref_config_data=config_data)
yield heartbeat('Marking runs as complete')
release_url = yield release_worker.RunsDoneWorkflow(
upload_build_id, upload_release_name, release_number)
yield heartbeat('Results viewable at: %s' % release_url)
def run(self,
image_pairs,
upload_build_id,
upload_release_name=None,
heartbeat=None,
run_name_depth=2):
if not upload_release_name:
upload_release_name = str(datetime.datetime.utcnow())
yield heartbeat('Creating release %s' % upload_release_name)
release_number = yield release_worker.CreateReleaseWorkflow(
upload_build_id, upload_release_name, upload_release_name)
config_dict = {
'viewportSize': {
'width': 1280,
'height': 1024,
}
}
if FLAGS.inject_css:
config_dict['injectCss'] = FLAGS.inject_css
if FLAGS.inject_js:
config_dict['injectJs'] = FLAGS.inject_js
config_data = json.dumps(config_dict)
yield heartbeat('Requesting captures')
requests = []
for old_img, new_img in image_pairs.iteritems():
yield heartbeat(old_img)
yield heartbeat(new_img)
yield heartbeat('--------------------------------')
url_parts = urlparse.urlparse(new_img)
url = (url_parts.path or '/').split('/')
run_name = url.pop()
for depth in range(run_name_depth - 1):
run_name = url.pop() + '/' + run_name
if len(url) <= 0:
break
run_name = urllib.url2pathname(run_name)
requests.append(
release_worker.RequestRunWorkflow(upload_build_id,
upload_release_name,
release_number,
run_name,
new_img,
config_data,
ref_url=old_img,
ref_config_data=config_data))
yield heartbeat('Processing requests ...')
yield requests
yield heartbeat('Marking runs as complete')
release_url = yield release_worker.RunsDoneWorkflow(
upload_build_id, upload_release_name, release_number)
yield heartbeat('Results viewable at: %s' % release_url)
def run(self,
start_url,
ignore_prefixes,
upload_build_id,
upload_release_name=None,
heartbeat=None):
if not ignore_prefixes:
ignore_prefixes = []
pending_urls = set([clean_url(start_url)])
seen_urls = set()
good_urls = set()
yield heartbeat('Scanning for content')
limit_depth = FLAGS.crawl_depth >= 0
depth = 0
while (not limit_depth or depth <= FLAGS.crawl_depth) and pending_urls:
# TODO: Enforce a job-wide timeout on the whole process of
# URL discovery, to make sure infinitely deep sites do not
# cause this job to never stop.
seen_urls.update(pending_urls)
yield heartbeat('Scanning %d pages for good urls' %
len(pending_urls))
output = yield [fetch_worker.FetchItem(u) for u in pending_urls]
pending_urls.clear()
for item in output:
if not item.data:
logging.debug('No data from url=%r', item.url)
continue
if item.headers.gettype() != 'text/html':
logging.debug('Skipping non-HTML document url=%r',
item.url)
continue
good_urls.add(item.url)
found = extract_urls(item.url, item.data)
pruned = prune_urls(found, start_url, [start_url],
ignore_prefixes)
new = pruned - seen_urls
pending_urls.update(new)
yield heartbeat('Found %d new URLs from %s' %
(len(new), item.url))
yield heartbeat('Finished crawl at depth %d' % depth)
depth += 1
yield heartbeat('Found %d total URLs, %d good HTML pages; starting '
'screenshots' % (len(seen_urls), len(good_urls)))
# TODO: Make the default release name prettier.
if not upload_release_name:
upload_release_name = str(datetime.datetime.utcnow())
release_number = yield release_worker.CreateReleaseWorkflow(
upload_build_id, upload_release_name, start_url)
run_requests = []
for url in good_urls:
yield heartbeat('Requesting run for %s' % url)
parts = urlparse.urlparse(url)
run_name = parts.path
config_dict = {
'viewportSize': {
'width': 1280,
'height': 1024,
}
}
if FLAGS.inject_css:
config_dict['injectCss'] = FLAGS.inject_css
if FLAGS.inject_js:
config_dict['injectJs'] = FLAGS.inject_js
if FLAGS.cookies:
config_dict['cookies'] = json.loads(open(FLAGS.cookies).read())
config_data = json.dumps(config_dict)
run_requests.append(
release_worker.RequestRunWorkflow(upload_build_id,
upload_release_name,
release_number, run_name,
url, config_data))
yield run_requests
yield heartbeat('Marking runs as complete')
release_url = yield release_worker.RunsDoneWorkflow(
upload_build_id, upload_release_name, release_number)
yield heartbeat('Results viewable at: %s' % release_url)