Exemple #1
0
def pubsub(instanceId=None):
    jobs = None
    output = ''
    # We keep all jobs for 10 hours
    redis_conn.zremrangebyscore(
        'jobs:{}'.format(instanceId),
        0,
        time.time() - 36000
    )

    if instanceId is not None:
        jobs = redis_conn.zrevrange(
            'jobs:{}'.format(instanceId), 0, 0, withscores=True
        )
        if jobs:
            for userId, job in jobs:
                    console = _read_console(job)
                    output += console\
                        .replace('\n', '<br />')\
                        .replace('#BEGIN#', '')\
                        .replace('#END#', '')
                    output = ansiconv.to_html(output)
                    # if '#END#' in console:
                    # Save job into the db auditlog

    return Response('data: {}\n\n'.format(output),
                    mimetype='text/event-stream')
Exemple #2
0
def pubsub(instanceId=None):
    jobs = None
    output = ''
    # We keep all jobs for 10 hours
    # get all jobs for an instance
    jobs_key = 'jobs:{}'.format(instanceId)
    jobs = redis_conn.hkeys(jobs_key)
    # sort by jobid/time most recent first
    jobs.sort(reverse=True)
    # filter older jobs
    old_jobs = filter(
        lambda x: x < (time.time() - 3600), jobs)
    # delete them
    if old_jobs:
        redis_conn.hdel(jobs_key, *old_jobs)

    if instanceId is not None and jobs:
        # timeStarted = jobs[0]
        # timeStarted = jobs[0]
        # userId = redis_conn.hget(
        #     jobs_key,
        #     timeStarted)
        # for job in jobs:
        job = jobs[0]
        console = _read_console(job)
        output += console\
            .replace('\n', '<br />')\
            .replace('#BEGIN#', '')\
            .replace('#END#', '')
        output = ansiconv.to_html(output)
        # if '#END#' in console:
        # Save job into the db auditlog

    return Response('data: {}\n\n'.format(output),
                    mimetype='text/event-stream')
Exemple #3
0
def pubsub(instanceId=None):
    jobs = None
    output = ''
    # We keep all jobs for 10 hours
    redis_conn.zremrangebyscore('jobs:{}'.format(instanceId), 0,
                                time.time() - 36000)

    if instanceId is not None:
        jobs = redis_conn.zrevrange('jobs:{}'.format(instanceId),
                                    0,
                                    0,
                                    withscores=True)
        if jobs:
            for userId, job in jobs:
                console = _read_console(job)
                output += console\
                    .replace('\n', '<br />')\
                    .replace('#BEGIN#', '')\
                    .replace('#END#', '')
                output = ansiconv.to_html(output)
                # if '#END#' in console:
                # Save job into the db auditlog

    return Response('data: {}\n\n'.format(output),
                    mimetype='text/event-stream')
Exemple #4
0
def start_task(message):
    time.sleep(1)
    project = Project.objects.get(id=message.content['project_id'])
    deployment = Deployment.objects.get(id=message.content['deployment_id'])
    deployment.output = ''
    deployment.save()

    engine = import_module(settings.SESSION_ENGINE)
    SessionStore = engine.SessionStore
    session = SessionStore(message.content['session_key'])

    if backend.get_task_details(project, deployment.task.name) is None:
        return

    process = subprocess.Popen(
        backend.build_command(project, deployment, session),
        stdout=subprocess.PIPE,
        stderr=subprocess.STDOUT,
        shell=True,
        executable=getattr(settings, 'SHELL', '/bin/sh'),
    )

    while True:
        nextline = process.stdout.readline()
        if nextline == '' and process.poll() is not None:
            break

        Group("deployment-{}".format(deployment.id)).send({
            "text":
            json.dumps({
                'status':
                'pending',
                'text':
                str('<span class="output-line">{}</span>'.format(
                    ansiconv.to_html(nextline)))
            }),
        })

        deployment.add_output(nextline)

        sys.stdout.flush()

    Deployment.objects.filter(pk=deployment.id).update(
        status=deployment.SUCCESS if process.returncode ==
        0 else deployment.FAILED)

    Group("deployment-{}".format(deployment.id)).send({
        "text":
        json.dumps({
            'status':
            deployment.SUCCESS
            if process.returncode == 0 else deployment.FAILED,
            'text':
            ''
        }),
    })

    deployment_finished.send(deployment, deployment_id=deployment.pk)
Exemple #5
0
def base64_decode(raw_id_1, raw_id_2):
    std_str = '<!DOCTYPE html><html lang="en"><head><meta charset="UTF-8"><link rel="stylesheet" href="https://cdn.jsdelivr.net/gh/tonsky/[email protected]/distr/fira_code.css"><style>.code-font-medium {font-family: \'Fira Code\', monospace;font-size: medium;}</style><style>'
    try:
        str_raw_html = base64.b64decode((''.join(open('static/datalogs/{}_{}.txt'.format(raw_id_1, raw_id_2), 'r').readlines())).encode()).decode().replace('\n', '<br>')
        html_conv = ansiconv.to_html(str_raw_html).replace('ansi37', '')
        str_html = '<body class="code-font-medium">{}</body></html>'.format(html_conv)
        st_css = ansiconv.base_css()
        return std_str + st_css + '</style></head>' + str_html
    except Exception as ident:
        return 'Error occurred. {}'.format(ident)
Exemple #6
0
 def __init__(self, title, details, parent = None):
     super(self.__class__, self).__init__(parent, QtCore.Qt.WindowSystemMenuHint | QtCore.Qt.WindowTitleHint | QtCore.Qt.MSWindowsFixedSizeDialogHint)
     self.setupUi(self)  # This is defined in design.py file automatically
                         # It sets up layout and widgets that are defined
     
     self.setWindowTitle("IEEE Testbank Tool - " + title)
     
     # Pre-processing
     details = str(details).replace("\n", "<br />\n")
     
     self.detailsTxt.setText("<style>%s</style>%s" % (ansiconv.base_css(), ansiconv.to_html(details)))
Exemple #7
0
def get_experiment(path):
    logdir = APP.config['logdir']
    complete_path = os.path.join(logdir, path)
    if os.path.isfile(complete_path):
        file_content = get_file(complete_path)
        if path.endswith(".log"):
            result = ansiconv.to_html(file_content)
        elif path.endswith(".ini"):
            result = html.escape(file_content)
        else:
            result = "Unknow file type: '{}'.".format(complete_path)
    else:
        result = "File '{}' does not exist.".format(complete_path)
    return Response(result, mimetype='text/html', status=200)
Exemple #8
0
def get_experiment(path):
    logdir = APP.config['logdir']
    complete_path = os.path.join(logdir, path)
    if os.path.isfile(complete_path):
        file_content = get_file(complete_path)
        if path.endswith(".log"):
            result = ansiconv.to_html(file_content)
        elif path.endswith(".ini"):
            result = file_content
        else:
            result = "Unknow file type: \"{}\".".format(complete_path)
    else:
        result = "File \"{}\" does not exist.".format(complete_path)
    return Response(result, mimetype='text/html', status=200)
Exemple #9
0
    def to_html(self):
        import ansiconv
        output = """
            <style>
              {}

              div.timelinetool {{
                font-family: monospace;
                white-space: pre-wrap;
                padding: 10px;
              }}
            </style>
            <div class="timelinetool ansi_fore ansi_back">{}</div>
            """.format(ansiconv.base_css(), ansiconv.to_html(str(self)))
        return output
Exemple #10
0
    def to_html(self):
        import ansiconv
        output = """
            <style>
              {}

              div.timelinetool {{
                font-family: monospace;
                white-space: pre-wrap;
                padding: 10px;
              }}
            </style>
            <div class="timelinetool ansi_fore ansi_back">{}</div>
            """.format(ansiconv.base_css(), ansiconv.to_html(str(self)))
        return output
Exemple #11
0
def get_experiment(path):
    logdir = APP.config['logdir']
    complete_path = os.path.join(logdir, path)
    if os.path.isfile(complete_path):
        file_content = get_file(complete_path)
        if path.endswith(".log"):
            result = ansiconv.to_html(html.escape(file_content))
        elif path.endswith(".ini"):
            lexer = IniLexer()
            formatter = HtmlFormatter(linenos=True)
            result = highlight(file_content, lexer, formatter)
        else:
            result = "Unknown file type: '{}'.".format(complete_path)
    else:
        result = "File '{}' does not exist.".format(complete_path)
    return Response(result, mimetype='text/html', status=200)
Exemple #12
0
    def lastlog(self, lines=20, format='', html=True):
        handler = self._loghandler
        if format:
            formatter = ColoredFormatter(format, datefmt=self.time_format)
        else:
            formatter = ColoredFormatter(
                self.html_format if html else self.format,
                datefmt=self.time_format)

        rv = u'\n'.join([formatter.format(i) for i in handler.buffer[-lines:]])

        if html:
            rv = ansiconv.to_html(self.html_fix(rv))
        else:
            rv = ansiconv.to_plain(rv)
        return rv
Exemple #13
0
    def get(self, user, repo, sha1):

        fullname = "%s/%s" % (user, repo)
        project = Project.get_by_id(fullname)

        if project is None:
            self.response.set_status(404)
            template = paths.JINJA_ENVIRONMENT.get_template('404.html')
            self.response.write(template.render(user=self.user,
                                                baseurl="../../../",
                                                message="Project %s not found" % fullname))
            return

        push = Push.get_by_id(sha1)
        if push is None:
            self.response.set_status(404)
            template = paths.JINJA_ENVIRONMENT.get_template('404.html')
            self.response.write(template.render(user=self.user,
                                                baseurl="../../../",
                                                message="Commit %s not found" % sha1))
            return

        repo = project.repo.get()
        junit = JUnitTestResult.query(JUnitTestResult.push==push.key).fetch()
        junit = [j for j in junit if j.valid]
        logging.info("Found %d junit results for sha %s", len(junit), sha1)
        junit = junit[0]

        sim_job = junit.travis_job.get()

        sim_logfiles = []

        for logfile in sim_job.logfiles:
            gcs_file = gcs.open(logfile)
            sim_logfiles.append((logfile.split('/')[-1].split(".")[0], ansiconv.to_html(gcs_file.read())))
            gcs_file.close()

        template = paths.JINJA_ENVIRONMENT.get_template('build.html')
        self.response.write(template.render(user=self.user,
                                            project=project,
                                            repo=repo,
                                            push=push,
                                            junit=junit,
                                            sim_job=sim_job,
                                            sim_logfiles=sim_logfiles))
        return
Exemple #14
0
    def output_stream_generator(self):
        if backend.get_task_details(self.project,
                                    self.object.task.name) is None:
            return

        try:
            process = subprocess.Popen(
                backend.build_command(self.project, self.object,
                                      self.request.session),
                stdout=subprocess.PIPE,
                stderr=subprocess.STDOUT,
                shell=True,
                executable=getattr(settings, 'SHELL', '/bin/sh'),
            )

            all_output = ''
            yield '<link rel="stylesheet" type="text/css" href="/static/css/console-style.css">'
            while True:
                nextline = process.stdout.readline()
                if nextline == '' and process.poll() is not None:
                    break

                all_output += nextline
                nextline = '<span class="output-line">{}</span>'.format(
                    ansiconv.to_html(nextline))
                yield nextline + ' ' * 1024

                sys.stdout.flush()

            self.object.status = self.object.SUCCESS if process.returncode == 0 else self.object.FAILED

            yield '<span id="finished" style="display:none;">{}</span> {}'.format(
                self.object.status, ' ' * 1024)

            self.object.output = all_output
            self.object.save()

            deployment_finished.send(self.object, deployment_id=self.object.pk)

        except Exception as e:
            message = "An error occurred: " + e.message
            yield '<span class="output-line">{}</span>'.format(
                message) + ' ' * 1024
            yield '<span id="finished" style="display:none;">failed</span> {}'.format(
                '*1024')
Exemple #15
0
def main(cfg):
    # get working directory
    wd = cfg.get(SETTINGS, 'repo_path')

    # perform git add
    cmd = cfg.get(SETTINGS, 'git_cmd_add').split(' ')
    p = run(cmd, cwd=wd, stdout=PIPE)
    print('*** git add done')

    # perform git diff
    cmd = cfg.get(SETTINGS, 'git_cmd_diff').split(' ')
    p = run(cmd, cwd=wd, stdout=PIPE)
    print('*** git diff done')
    diffo = p.stdout.decode('utf-8')

    # only commit and report if there is a diff output
    if len(diffo) < DIFF_LEN_REQUIRED:
        print('*** no changes detected, will stop here')
        return False
    else:
        print('*** changes detected (%d bytes), will proceed' % len(diffo)) 

    # perform git commit
    cmd = cfg.get(SETTINGS, 'git_cmd_commit').split(' ')
    p = run(cmd, cwd=wd, stdout=PIPE)
    print('*** commited')

    # perform redacting
    if cfg.getboolean(SETTINGS, 'redact_enable'):
        cre = re.compile(cfg.get(SETTINGS, 'redact_pattern'), flags=re.MULTILINE)
        replace = cfg.get(SETTINGS, 'redact_replace')
        print('*** perfoming redaction')
        diffr = re.sub(cre, replace, diffo)

    # remove diff and index lines
    if cfg.getboolean(SETTINGS, 'redact_minimize'):
        cre = re.compile(r'(diff\s--git\s.+\n|index\s.+\n|---\s.+\n)', flags=re.MULTILINE)
        diffr = re.sub(cre, '', diffr)

    # send report
    report(cfg, ansiconv.to_html(diffr))
    print('*** report done')
Exemple #16
0
    def output_stream_generator(self):
        if get_task_details(self.project, self.object.task.name) is None:
            return

        try:
            process = subprocess.Popen(
                build_command(self.object, self.request.session),
                stdout=subprocess.PIPE,
                stderr=subprocess.STDOUT,
                shell=True,
                executable=getattr(settings, 'SHELL', '/bin/sh'),
            )

            all_output = ''
            yield '<link rel="stylesheet" type="text/css" href="/static/css/console-style.css">'
            while True:
                nextline = process.stdout.readline()
                if nextline == '' and process.poll() is not None:
                    break

                all_output += nextline
                nextline = '<span class="output-line">{}</span>'.format(ansiconv.to_html(nextline))
                yield nextline + ' '*1024

                sys.stdout.flush()

            self.object.status = self.object.SUCCESS if process.returncode == 0 else self.object.FAILED

            yield '<span id="finished" style="display:none;">{}</span> {}'.format(self.object.status, ' '*1024)

            self.object.output = all_output
            self.object.save()

            deployment_finished.send(self.object, deployment_id=self.object.pk)

        except Exception as e:
            message = "An error occurred: " + e.message
            yield '<span class="output-line">{}</span>'.format(message) + ' '*1024
            yield '<span id="finished" style="display:none;">failed</span> {}'.format('*1024')
Exemple #17
0
def start_task(message):
    time.sleep(1)
    project = Project.objects.get(id=message.content['project_id'])
    deployment = Deployment.objects.get(id=message.content['deployment_id'])
    deployment.output = ''
    deployment.save()

    engine = import_module(settings.SESSION_ENGINE)
    SessionStore = engine.SessionStore
    session = SessionStore(message.content['session_key'])

    if backend.get_task_details(project, deployment.task.name) is None:
        return

    process = subprocess.Popen(
        backend.build_command(project, deployment, session),
        stdout=subprocess.PIPE,
        stderr=subprocess.STDOUT,
        stdin=subprocess.PIPE,
        shell=True,
        executable=getattr(settings, 'SHELL', '/bin/sh'),
        close_fds=True
    )

    fd = process.stdout.fileno()
    fl = fcntl.fcntl(fd, fcntl.F_GETFL)
    fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NONBLOCK)

    while True:
        try:
            nextline = process.stdout.readline()
        except IOError as e:
            nextline = ''

        if nextline == '' and process.poll() is not None:
            break
        #
        # next_input = deployment.get_next_input()
        # if next_input:
        #     process.stdin.write(next_input + '\n')

        if nextline:
            Group("deployment-{}".format(deployment.id)).send({
                "text": json.dumps({
                    'status': 'pending',
                    'text': str('<span class="output-line">{}</span>'.format(ansiconv.to_html(nextline)))
                }),
            }, immediately=True)

            deployment.add_output(nextline)

        sys.stdout.flush()

    Deployment.objects.filter(pk=deployment.id).update(
        status=deployment.SUCCESS if process.returncode == 0 else deployment.FAILED
    )

    Group("deployment-{}".format(deployment.id)).send({
        "text": json.dumps({
            'status': deployment.SUCCESS if process.returncode == 0 else deployment.FAILED,
            'text': ''
        }),
    }, immediately=True)

    deployment_finished.send(deployment, deployment_id=deployment.pk)
Exemple #18
0
def test_basic_to_html_replace_newline():
    expected = '<span class="ansi32">Foo<br />\nBar</span>'
    actual = ansiconv.to_html('\033[0;32mFoo\nBar', replace_newline=True)
    assert actual == expected
Exemple #19
0
 def get_context_data(self, **kwargs):
     context = super(DeploymentDetail, self).get_context_data(**kwargs)
     context['deploy_output'] = ansiconv.to_html(self.object.output) if self.object.output else ''
     return context
Exemple #20
0
def test_to_html_with_cursor_up_command():
    expected = '<span class="ansi34">Foo</span>'
    actual = ansiconv.to_html('\033[0;30mFoo\n\033[A\033[0;34mFoo')
    assert actual == expected
Exemple #21
0
def test_to_html_with_multiple_color_changes():
    expected = '<span class="ansi32">Foo</span><span class="ansi31">Bar</span>'
    actual = ansiconv.to_html('\033[0;32mFoo\033[0;31mBar')
    assert actual == expected
Exemple #22
0
def test_basic_to_html():
    expected = '<span class="ansi32">Foo</span>'
    actual = ansiconv.to_html('\033[0;32mFoo')
    assert actual == expected
Exemple #23
0
 def emit(self, record):
     super().emit(record)
     service.most_recent_line = ansiconv.to_html(
         service.html_fix(html_formatter.format(record))) + '\n'
Exemple #24
0
 def get_formatted_output(self):
     return ansiconv.to_html(self.output) if self.output else ''
Exemple #25
0
 def get_formatted_output(self):
     return ansiconv.to_html(self.output) if self.output else ''
    def log(id):

        testroles_row = testroles.query.filter(testroles.id == id).order_by(testroles.id).first()
        html = ansiconv.to_html(testroles_row.testrole_log)
        css = ansiconv.base_css()
        return render_template('logs.html', testroles_row=testroles_row, html=html, css=css)
Exemple #27
0
    def retrieve(self, request, *args, **kwargs):
        job = self.get_object()
        try:
            target_format = request.accepted_renderer.format
            if target_format in ('html', 'api', 'json'):
                content_format = request.query_params.get(
                    'content_format', 'html')
                content_encoding = request.query_params.get(
                    'content_encoding', None)
                start_line = request.query_params.get('start_line', 0)
                end_line = request.query_params.get('end_line', None)
                dark_val = request.query_params.get('dark', '')
                dark = bool(dark_val
                            and dark_val[0].lower() in ('1', 't', 'y'))
                content_only = bool(target_format in ('api', 'json'))
                dark_bg = (content_only and dark) or (not content_only and
                                                      (dark or not dark_val))
                content, start, end, absolute_end = job.result_stdout_raw_limited(
                    start_line, end_line)

                # Remove any ANSI escape sequences containing job event data.
                content = re.sub(
                    r'\x1b\[K(?:[A-Za-z0-9+/=]+\x1b\[\d+D)+\x1b\[K', '',
                    content)

                body = ansiconv.to_html(cgi.escape(content))

                context = {
                    'title': get_view_name(self.__class__),
                    'body': mark_safe(body),
                    'dark': dark_bg,
                    'content_only': content_only,
                }
                data = render_to_string('api/stdout.html', context).strip()

                if target_format == 'api':
                    return Response(mark_safe(data))
                if target_format == 'json':
                    if content_encoding == 'base64' and content_format == 'ansi':
                        return Response({
                            'range': {
                                'start': start,
                                'end': end,
                                'absolute_end': absolute_end
                            },
                            'content':
                            b64encode(content.encode('utf-8'))
                        })
                    elif content_format == 'html':
                        return Response({
                            'range': {
                                'start': start,
                                'end': end,
                                'absolute_end': absolute_end
                            },
                            'content': body
                        })
                return Response(data)
            elif target_format == 'txt':
                return Response(job.result_stdout)
            elif target_format == 'ansi':
                return Response(job.result_stdout_raw)
            elif target_format in {'txt_download', 'ansi_download'}:
                filename = '{type}_{pk}{suffix}.txt'.format(
                    type=camelcase_to_underscore(job.__class__.__name__),
                    pk=job.id,
                    suffix='.ansi' if target_format == 'ansi_download' else '')
                content_fd = job.result_stdout_raw_handle(
                    enforce_max_bytes=False)
                if target_format == 'txt_download':
                    content_fd = StdoutANSIFilter(content_fd)
                response = HttpResponse(FileWrapper(content_fd),
                                        content_type='text/plain')
                response[
                    "Content-Disposition"] = 'attachment; filename="{}"'.format(
                        filename)
                return response
            else:
                return super(JobStdout, self).retrieve(request, *args,
                                                       **kwargs)
        except StdoutMaxBytesExceeded as e:
            response_message = _(
                "Standard Output too large to display ({text_size} bytes), "
                "only download supported for sizes over {supported_size} bytes."
            ).format(text_size=e.total, supported_size=e.supported)
            if request.accepted_renderer.format == 'json':
                return Response({
                    'range': {
                        'start': 0,
                        'end': 1,
                        'absolute_end': 1
                    },
                    'content': response_message
                })
            else:
                return Response(response_message)
Exemple #28
0
 def get_context_data(self, **kwargs):
     context = super(DeploymentDetail, self).get_context_data(**kwargs)
     context['deploy_output'] = ansiconv.to_html(
         self.object.output) if self.object.output else ''
     return context