예제 #1
0
    def run(self,
            release_url,
            tests,
            upload_build_id,
            upload_release_name,
            heartbeat=None):
        if not upload_release_name:
            upload_release_name = str(datetime.datetime.utcnow())

        yield heartbeat('Creating release %s' % upload_release_name)
        release_number = yield release_worker.CreateReleaseWorkflow(
            upload_build_id, upload_release_name, release_url)

        pending_uploads = []
        for test in tests:
            item = release_worker.RequestRunWorkflow(
                upload_build_id,
                upload_release_name,
                release_number,
                test.name,
                url=test.run_url,
                config_data=test.run_config_data,
                ref_url=test.ref_url,
                ref_config_data=test.ref_config_data)
            pending_uploads.append(item)

        yield heartbeat('Requesting %d runs' % len(pending_uploads))
        yield pending_uploads

        yield heartbeat('Marking runs as complete')
        release_url = yield release_worker.RunsDoneWorkflow(
            upload_build_id, upload_release_name, release_number)

        yield heartbeat('Results viewable at: %s' % release_url)
예제 #2
0
    def run(self,
            release_url,
            tests,
            upload_build_id,
            upload_release_name,
            heartbeat=None):
        if not upload_release_name:
            upload_release_name = str(datetime.datetime.now())

        yield heartbeat('Creating release %s' % upload_release_name)
        release_number = yield release_worker.CreateReleaseWorkflow(
            upload_build_id, upload_release_name, release_url)

        pending_uploads = []
        for test in tests:
            item = RunTest(upload_build_id,
                           upload_release_name,
                           release_number,
                           test,
                           heartbeat=heartbeat)
            pending_uploads.append(item)

        yield heartbeat('Uploading %d runs' % len(pending_uploads))
        yield pending_uploads

        yield heartbeat('Marking runs as complete')
        release_url = yield release_worker.RunsDoneWorkflow(
            upload_build_id, upload_release_name, release_number)

        yield heartbeat('Results viewable at: %s' % release_url)
예제 #3
0
    def run(self,
            new_url,
            baseline_url,
            upload_build_id,
            upload_release_name=None,
            heartbeat=None):
        # TODO: Make the default release name prettier.
        if not upload_release_name:
            upload_release_name = str(datetime.datetime.utcnow())

        yield heartbeat('Creating release %s' % upload_release_name)
        release_number = yield release_worker.CreateReleaseWorkflow(
            upload_build_id, upload_release_name, new_url)

        config_dict = {
            'viewportSize': {
                'width': 1280,
                'height': 1024,
            }
        }

        if FLAGS.width:
            config_dict['viewportSize']['width'] = FLAGS.width
        if FLAGS.height:
            config_dict['viewportSize']['height'] = FLAGS.height

        if FLAGS.inject_css:
            config_dict['injectCss'] = FLAGS.inject_css
        if FLAGS.inject_js:
            config_dict['injectJs'] = FLAGS.inject_js

        if FLAGS.http_username:
            config_dict['httpUserName'] = FLAGS.http_username
        if FLAGS.http_password:
            config_dict['httpPassword'] = FLAGS.http_password

        config_data = json.dumps(config_dict)

        url_parts = urlparse.urlparse(new_url)

        yield heartbeat('Requesting captures')
        yield release_worker.RequestRunWorkflow(upload_build_id,
                                                upload_release_name,
                                                release_number,
                                                url_parts.path or '/',
                                                new_url,
                                                config_data,
                                                ref_url=baseline_url,
                                                ref_config_data=config_data)

        yield heartbeat('Marking runs as complete')
        release_url = yield release_worker.RunsDoneWorkflow(
            upload_build_id, upload_release_name, release_number)

        yield heartbeat('Results viewable at: %s' % release_url)
예제 #4
0
    def run(self,
            image_pairs,
            upload_build_id,
            upload_release_name=None,
            heartbeat=None,
            run_name_depth=2):

        if not upload_release_name:
            upload_release_name = str(datetime.datetime.utcnow())

        yield heartbeat('Creating release %s' % upload_release_name)
        release_number = yield release_worker.CreateReleaseWorkflow(
            upload_build_id, upload_release_name, upload_release_name)

        config_dict = {
            'viewportSize': {
                'width': 1280,
                'height': 1024,
            }
        }
        if FLAGS.inject_css:
            config_dict['injectCss'] = FLAGS.inject_css
        if FLAGS.inject_js:
            config_dict['injectJs'] = FLAGS.inject_js
        config_data = json.dumps(config_dict)

        yield heartbeat('Requesting captures')
        requests = []
        for old_img, new_img in image_pairs.iteritems():

            yield heartbeat(old_img)
            yield heartbeat(new_img)
            yield heartbeat('--------------------------------')

            url_parts = urlparse.urlparse(new_img)
            url = (url_parts.path or '/').split('/')
            run_name = url.pop()
            for depth in range(run_name_depth - 1):
                run_name = url.pop() + '/' + run_name
                if len(url) <= 0:
                    break
            run_name = urllib.url2pathname(run_name)

            requests.append(
                release_worker.RequestRunWorkflow(upload_build_id,
                                                  upload_release_name,
                                                  release_number,
                                                  run_name,
                                                  new_img,
                                                  config_data,
                                                  ref_url=old_img,
                                                  ref_config_data=config_data))

        yield heartbeat('Processing requests ...')
        yield requests

        yield heartbeat('Marking runs as complete')
        release_url = yield release_worker.RunsDoneWorkflow(
            upload_build_id, upload_release_name, release_number)

        yield heartbeat('Results viewable at: %s' % release_url)
예제 #5
0
    def run(self,
            start_url,
            ignore_prefixes,
            upload_build_id,
            upload_release_name=None,
            heartbeat=None):
        if not ignore_prefixes:
            ignore_prefixes = []

        pending_urls = set([clean_url(start_url)])
        seen_urls = set()
        good_urls = set()

        yield heartbeat('Scanning for content')

        limit_depth = FLAGS.crawl_depth >= 0
        depth = 0
        while (not limit_depth or depth <= FLAGS.crawl_depth) and pending_urls:
            # TODO: Enforce a job-wide timeout on the whole process of
            # URL discovery, to make sure infinitely deep sites do not
            # cause this job to never stop.
            seen_urls.update(pending_urls)
            yield heartbeat('Scanning %d pages for good urls' %
                            len(pending_urls))
            output = yield [fetch_worker.FetchItem(u) for u in pending_urls]
            pending_urls.clear()

            for item in output:
                if not item.data:
                    logging.debug('No data from url=%r', item.url)
                    continue

                if item.headers.gettype() != 'text/html':
                    logging.debug('Skipping non-HTML document url=%r',
                                  item.url)
                    continue

                good_urls.add(item.url)
                found = extract_urls(item.url, item.data)
                pruned = prune_urls(found, start_url, [start_url],
                                    ignore_prefixes)
                new = pruned - seen_urls
                pending_urls.update(new)
                yield heartbeat('Found %d new URLs from %s' %
                                (len(new), item.url))

            yield heartbeat('Finished crawl at depth %d' % depth)
            depth += 1

        yield heartbeat('Found %d total URLs, %d good HTML pages; starting '
                        'screenshots' % (len(seen_urls), len(good_urls)))

        # TODO: Make the default release name prettier.
        if not upload_release_name:
            upload_release_name = str(datetime.datetime.utcnow())

        release_number = yield release_worker.CreateReleaseWorkflow(
            upload_build_id, upload_release_name, start_url)

        run_requests = []
        for url in good_urls:
            yield heartbeat('Requesting run for %s' % url)
            parts = urlparse.urlparse(url)
            run_name = parts.path

            config_dict = {
                'viewportSize': {
                    'width': 1280,
                    'height': 1024,
                }
            }
            if FLAGS.inject_css:
                config_dict['injectCss'] = FLAGS.inject_css
            if FLAGS.inject_js:
                config_dict['injectJs'] = FLAGS.inject_js
            if FLAGS.cookies:
                config_dict['cookies'] = json.loads(open(FLAGS.cookies).read())
            config_data = json.dumps(config_dict)

            run_requests.append(
                release_worker.RequestRunWorkflow(upload_build_id,
                                                  upload_release_name,
                                                  release_number, run_name,
                                                  url, config_data))

        yield run_requests

        yield heartbeat('Marking runs as complete')
        release_url = yield release_worker.RunsDoneWorkflow(
            upload_build_id, upload_release_name, release_number)

        yield heartbeat('Results viewable at: %s' % release_url)