Ejemplo n.º 1
0
 def _get_and_retry(self, url, **kwargs):
     """
     requests 2.5.0 HTTPS IS A LITTLE UNSTABLE
     """
     kwargs = set_default(kwargs, {"timeout": self.timeout.seconds})
     try:
         return http.get(url, **kwargs)
     except Exception, e:
         try:
             Thread.sleep(seconds=5)
             return http.get(url.replace("https://", "http://"), **kwargs)
         except Exception, f:
             Log.error("Tried {{url}} twice.  Both failed.", {"url": url}, cause=[e, f])
Ejemplo n.º 2
0
 def _get_and_retry(self, url, **kwargs):
     """
     requests 2.5.0 HTTPS IS A LITTLE UNSTABLE
     """
     kwargs = set_default(kwargs, {"timeout": self.timeout.seconds})
     try:
         return http.get(url, **kwargs)
     except Exception, e:
         try:
             Thread.sleep(seconds=5)
             return http.get(url.replace("https://", "http://"), **kwargs)
         except Exception, f:
             Log.error("Tried {{url}} twice.  Both failed.", {"url": url},
                       cause=[e, f])
Ejemplo n.º 3
0
    def test_branch_count(self):
        if self.not_real_service():
            return

        test = wrap({"query": {
            "from": {
                "type": "elasticsearch",
                "settings": {
                    "host": ES_CLUSTER_LOCATION,
                    "index": "unittest",
                    "type": "test_result"
                }
            },
            "select": [
                {"aggregate": "count"},
            ],
            "edges": [
                "build.branch"
            ],
            "where": {"or": [
                {"missing": "build.id"}
                # {"gte": {"timestamp": Date.floor(Date.now() - (Duration.DAY * 7), Duration.DAY).milli / 1000}}
            ]},
            "format": "table"
        }})

        query = convert.unicode2utf8(convert.value2json(test.query))
        # EXECUTE QUERY
        with Timer("query"):
            response = http.get(self.service_url, data=query)
            if response.status_code != 200:
                error(response)
        result = convert.json2value(convert.utf82unicode(response.all_content))

        Log.note("result\n{{result|indent}}", {"result": result})
def verify_blobber_file(line_number, name, url):
    """
    :param line_number:  for debugging
    :param name:  for debugging
    :param url:  TO BE READ
    :return:  RETURNS BYTES **NOT** UNICODE
    """
    if name in ["emulator-5554.log", "qemu.log"] or any(map(name.endswith, [".png", ".html"])):
        return None, 0

    with Timer("Read {{name}}: {{url}}", {"name": name, "url": url}, debug=DEBUG):
        response = http.get(url)
        try:
            logs = response.all_lines
        except Exception, e:
            if name.endswith("_raw.log"):
                Log.error(
                    "Line {{line}}: {{name}} = {{url}} is NOT structured log",
                    line=line_number,
                    name=name,
                    url=url,
                    cause=e
                )
            if DEBUG:
                Log.note(
                    "Line {{line}}: {{name}} = {{url}} is NOT structured log",
                    line=line_number,
                    name=name,
                    url=url
                )
            return None, 0
def verify_blobber_file(line_number, name, url):
    """
    :param line_number:  for debugging
    :param name:  for debugging
    :param url:  TO BE READ
    :return:  RETURNS BYTES **NOT** UNICODE
    """
    if name in ["emulator-5554.log", "qemu.log"] or any(
            map(name.endswith, [".png", ".html"])):
        return None, 0

    with Timer("Read {{name}}: {{url}}", {
            "name": name,
            "url": url
    },
               debug=DEBUG):
        response = http.get(url)
        try:
            logs = response.all_lines
        except Exception, e:
            if name.endswith("_raw.log"):
                Log.error(
                    "Line {{line}}: {{name}} = {{url}} is NOT structured log",
                    line=line_number,
                    name=name,
                    url=url,
                    cause=e)
            if DEBUG:
                Log.note(
                    "Line {{line}}: {{name}} = {{url}} is NOT structured log",
                    line=line_number,
                    name=name,
                    url=url)
            return None, 0
Ejemplo n.º 6
0
    def test_multiple_agg_on_same_field(self):
        if self.not_real_service():
            return

        test = wrap({
            "query": {
                "from": {
                    "type": "elasticsearch",
                    "settings": {
                        "host": ES_CLUSTER_LOCATION,
                        "index": "unittest",
                        "type": "test_result"
                    }
                },
                "select": [{
                    "name": "max_bytes",
                    "value": "run.stats.bytes",
                    "aggregate": "max"
                }, {
                    "name": "count",
                    "value": "run.stats.bytes",
                    "aggregate": "count"
                }]
            }
        })

        query = unicode2utf8(convert.value2json(test.query))
        # EXECUTE QUERY
        with Timer("query"):
            response = http.get(self.testing.query, data=query)
            if response.status_code != 200:
                error(response)
        result = json2value(utf82unicode(response.all_content))

        Log.note("result\n{{result|indent}}", {"result": result})
Ejemplo n.º 7
0
def _get_branches_from_hg(settings):
    # GET MAIN PAGE
    response = http.get(settings.url)
    doc = BeautifulSoup(response.all_content)

    all_repos = doc("table")[1]
    branches = UniqueIndex(["name", "locale"], fail_on_dup=False)
    for i, r in enumerate(all_repos("tr")):
        dir, name = [v.text.strip() for v in r("td")]

        b = _get_single_branch_from_hg(settings, name, dir.lstrip("/"))
        branches.extend(b)

    # branches.add(set_default({"name": "release-mozilla-beta"}, branches["mozilla-beta", DEFAULT_LOCALE]))
    for b in list(branches["mozilla-beta", ]):
        branches.add(set_default({"name": "release-mozilla-beta"}, b))  # THIS IS THE l10n "name"
        b.url = "https://hg.mozilla.org/releases/mozilla-beta"          # THIS IS THE

    for b in list(branches["mozilla-release", ]):
        branches.add(set_default({"name": "release-mozilla-release"}, b))

    for b in list(branches["mozilla-aurora", ]):
        if b.locale == "en-US":
            continue
        branches.add(set_default({"name": "comm-aurora"}, b))
        # b.url = "https://hg.mozilla.org/releases/mozilla-aurora"

    return branches
Ejemplo n.º 8
0
def get_branches(settings):
    # GET MAIN PAGE
    response = http.get(settings.url)
    doc = BeautifulSoup(response.all_content)

    all_repos = doc("table")[1]
    branches = UniqueIndex(["name", "locale"], fail_on_dup=False)
    for i, r in enumerate(all_repos("tr")):
        dir, name = [v.text.strip() for v in r("td")]

        b = get_branch(settings, name, dir.lstrip("/"))
        branches.extend(b)

    # branches.add(set_default({"name": "release-mozilla-beta"}, branches["mozilla-beta", DEFAULT_LOCALE]))
    for b in list(branches["mozilla-beta", ]):
        branches.add(set_default({"name": "release-mozilla-beta"}, b))

    for b in list(branches["mozilla-release", ]):
        branches.add(set_default({"name": "release-mozilla-release"}, b))

    for b in list(branches["mozilla-aurora", ]):
        if b.locale == "en-US":
            continue
        branches.add(set_default({"name": "comm-aurora"}, b))
    return branches
Ejemplo n.º 9
0
def _get_url(url, branch, **kwargs):
    with Explanation("get push from {{url}}", url=url):
        response = http.get(url, **kwargs)
        data = convert.json2value(response.content.decode("utf8"))
        if isinstance(data, basestring) and data.startswith("unknown revision"):
            Log.error("Unknown push {{revision}}", revision=strings.between(data, "'", "'"))
        branch.url = _trim(url)  #RECORD THIS SUCCESS IN THE BRANCH
        return data
Ejemplo n.º 10
0
def _get_url(url, branch, **kwargs):
    with Explanation("get push from {{url}}", url=url, debug=DEBUG):
        response = http.get(url, **kwargs)
        data = json2value(response.content.decode("utf8"))
        if isinstance(data, (text_type, str)) and data.startswith("unknown revision"):
            Log.error(UNKNOWN_PUSH, revision=strings.between(data, "'", "'"))
        branch.url = _trim(url)  # RECORD THIS SUCCESS IN THE BRANCH
        return data
Ejemplo n.º 11
0
def _get_url(url, branch, **kwargs):
    with Explanation("get push from {{url}}", url=url, debug=DEBUG):
        response = http.get(url, **kwargs)
        data = json2value(response.content.decode("utf8"))
        if isinstance(data, (text_type, str)) and data.startswith("unknown revision"):
            Log.error("Unknown push {{revision}}", revision=strings.between(data, "'", "'"))
        branch.url = _trim(url)  # RECORD THIS SUCCESS IN THE BRANCH
        return data
Ejemplo n.º 12
0
        def more():
            xml = http.get(self.url + "?" + value2url_param(state)).content
            data = BeautifulSoup(xml, 'xml')

            state.get_more = data.find("istruncated").contents[0] == "true"
            contents = data.findAll("contents")
            state.marker = contents[-1].find("key").contents[0]
            return [{k: t(d.find(k).contents[0]) for k, t in content_keys.items()} for d in contents]
Ejemplo n.º 13
0
Archivo: s3.py Proyecto: rv404674/TUID
        def more():
            xml = http.get(self.url + "?" + value2url_param(state)).content
            data = BeautifulSoup(xml, 'xml')

            state.get_more = data.find("istruncated").contents[0] == "true"
            contents = data.findAll("contents")
            state.marker = contents[-1].find("key").contents[0]
            return [{k: t(d.find(k).contents[0]) for k, t in content_keys.items()} for d in contents]
Ejemplo n.º 14
0
 def _find(b, please_stop):
     try:
         url = b.url + "rev/" + revision
         response = http.get(url)
         if response.status_code == 200:
             Log.note("{{revision}} found at {{url}}", url=url, revision=revision)
     except Exception, e:
         pass
Ejemplo n.º 15
0
    def _open(self):
        """ DO NOT USE THIS UNLESS YOU close() FIRST"""
        if self.settings.host.startswith("mysql://"):
            # DECODE THE URI: mysql://username:password@host:optional_port/database_name
            up = strings.between(self.settings.host, "mysql://", "@")
            if ":" in up:
                self.settings.username, self.settings.password = unquote(up).split(":")
            else:
                self.settings.username = up

            url = strings.between(self.settings.host, "@", None)
            hp, self.settings.schema = url.split("/", 1)
            if ":" in hp:
                self.settings.host, self.settings.port = hp.split(":")
                self.settings.port = int(self.settings.port)
            else:
                self.settings.host = hp

        # SSL PEM
        if self.settings.host in ("localhost", "mysql"):
            ssl_context = None
        else:
            if self.settings.ssl and not self.settings.ssl.pem:
                Log.error("Expecting 'pem' property in ssl")
            # ssl_context = ssl.create_default_context(**get_ssl_pem_file(self.settings.ssl.pem))
            filename = File(".pem") / URL(self.settings.ssl.pem).host
            filename.write_bytes(http.get(self.settings.ssl.pem).content)
            ssl_context = {"ca": filename.abspath}

        try:
            self.db = connect(
                host=self.settings.host,
                port=self.settings.port,
                user=coalesce(self.settings.username, self.settings.user),
                passwd=coalesce(self.settings.password, self.settings.passwd),
                db=coalesce(self.settings.schema, self.settings.db),
                read_timeout=coalesce(self.settings.read_timeout, (EXECUTE_TIMEOUT / 1000) - 10 if EXECUTE_TIMEOUT else None, 5*60),
                charset=u"utf8",
                use_unicode=True,
                ssl=ssl_context,
                cursorclass=cursors.SSCursor
            )
        except Exception as e:
            if self.settings.host.find("://") == -1:
                Log.error(
                    u"Failure to connect to {{host}}:{{port}}",
                    host=self.settings.host,
                    port=self.settings.port,
                    cause=e
                )
            else:
                Log.error(u"Failure to connect.  PROTOCOL PREFIX IS PROBABLY BAD", e)
        self.cursor = None
        self.partial_rollback = False
        self.transaction_level = 0
        self.backlog = []  # accumulate the write commands so they are sent at once
        if self.readonly:
            self.begin()
Ejemplo n.º 16
0
def get_http(ref, url):
    from pyLibrary.env import http

    params = url.query
    new_value = convert.json2value(http.get(ref),
                                   params=params,
                                   flexible=True,
                                   paths=True)
    return new_value
Ejemplo n.º 17
0
    def test_timing(self):
        if self.not_real_service():
            return

        test = wrap({
            "query": {
                "from": {
                    "type": "elasticsearch",
                    "settings": {
                        "host": ES_CLUSTER_LOCATION,
                        "index": "unittest",
                        "type": "test_result"
                    }
                },
                "select": [{
                    "name": "count",
                    "value": "run.duration",
                    "aggregate": "count"
                }, {
                    "name": "total",
                    "value": "run.duration",
                    "aggregate": "sum"
                }],
                "edges": [{
                    "name": "chunk",
                    "value": ["run.suite", "run.chunk"]
                }, "result.ok"],
                "where": {
                    "and": [{
                        "lt": {
                            "timestamp": Date.floor(Date.now()).milli / 1000
                        }
                    }, {
                        "gte": {
                            "timestamp":
                            Date.floor(Date.now() - (Duration.DAY * 7),
                                       Duration.DAY).milli / 1000
                        }
                    }]
                },
                "format":
                "cube",
                "samples": {
                    "limit": 30
                }
            }
        })

        query = unicode2utf8(convert.value2json(test.query))
        # EXECUTE QUERY
        with Timer("query"):
            response = http.get(self.testing.query, data=query)
            if response.status_code != 200:
                error(response)
        result = json2value(utf82unicode(response.all_content))

        Log.note("result\n{{result|indent}}", {"result": result})
Ejemplo n.º 18
0
 def _get_source_code_from_hg(self, revision, file_path):
     response = http.get(
         expand_template(
             FILE_URL, {
                 "location": revision.branch.url,
                 "rev": revision.changeset.id,
                 "path": file_path
             }))
     return response.content.decode("utf8", "replace")
Ejemplo n.º 19
0
 def _find(b, please_stop):
     try:
         url = b.url + "rev/" + revision
         response = http.get(url)
         if response.status_code == 200:
             Log.note("{{revision}} found at {{url}}",
                      url=url,
                      revision=revision)
     except Exception, e:
         pass
Ejemplo n.º 20
0
 def test_coverage_parser(self):
     diff = http.get(
         'https://hg.mozilla.org/mozilla-central/raw-rev/14dc6342ec5'
     ).content.decode('utf8')
     moves = diff_to_moves(diff)
     Log.note("{{files}}",
              files=[
                  m.old.name if m.new.name == 'dev/null' else m.new.name
                  for m in moves
              ])
Ejemplo n.º 21
0
def get_branch(settings, description, dir):
    if dir == "users":
        return []
    response = http.get(settings.url + "/" + dir)
    doc = BeautifulSoup(response.all_content)

    output = []
    try:
        all_branches = doc("table")[0]
    except Exception, _:
        return []
Ejemplo n.º 22
0
def _get_single_branch_from_hg(settings, description, dir):
    if dir == "users":
        return []
    response = http.get(settings.url + "/" + dir)
    doc = BeautifulSoup(response.all_content)

    output = []
    try:
        all_branches = doc("table")[0]
    except Exception, _:
        return []
Ejemplo n.º 23
0
 def _find(please_stop):
     for b in queue:
         if please_stop:
             return
         try:
             url = b.url + "json-info?node=" + revision
             response = http.get(url, timeout=30)
             if response.status_code == 200:
                 with locker:
                     output.append(b)
                 Log.note("{{revision}} found at {{url}}", url=url, revision=revision)
         except Exception, f:
             problems.append(f)
Ejemplo n.º 24
0
    def test_simple_query(self):
        if self.not_real_service():
            return

        query = convert.unicode2utf8(convert.value2json({"from": "unittest"}))
        # EXECUTE QUERY
        with Timer("query"):
            response = http.get(self.service_url, data=query)
            if response.status_code != 200:
                error(response)
        result = convert.json2value(convert.utf82unicode(response.all_content))

        Log.note("result\n{{result|indent}}", {"result": result})
Ejemplo n.º 25
0
        def inner(changeset_id):
            if self.es.cluster.version.startswith("1.7."):
                query = {
                    "query": {"filtered": {
                        "query": {"match_all": {}},
                        "filter": {"and": [
                            {"prefix": {"changeset.id": changeset_id}},
                            {"range": {"etl.timestamp": {"gt": MIN_ETL_AGE}}}
                        ]}
                    }},
                    "size": 1
                }
            else:
                query = {
                    "query": {"bool": {"must": [
                        {"prefix": {"changeset.id": changeset_id}},
                        {"range": {"etl.timestamp": {"gt": MIN_ETL_AGE}}}
                    ]}},
                    "size": 1
                }

            try:
                # ALWAYS TRY ES FIRST
                with self.es_locker:
                    response = self.es.search(query)
                    json_diff = response.hits.hits[0]._source.changeset.diff
                if json_diff:
                    return json_diff
            except Exception as e:
                pass

            url = expand_template(DIFF_URL, {"location": revision.branch.url, "rev": changeset_id})
            if DEBUG:
                Log.note("get unified diff from {{url}}", url=url)
            try:
                response = http.get(url)
                diff = response.content.decode("utf8", "replace")
                json_diff = diff_to_json(diff)
                num_changes = _count(c for f in json_diff for c in f.changes)
                if json_diff:
                    if num_changes < MAX_DIFF_SIZE:
                        return json_diff
                    elif revision.changeset.description.startswith("merge "):
                        return None  # IGNORE THE MERGE CHANGESETS
                    else:
                        Log.warning("Revision at {{url}} has a diff with {{num}} changes, ignored", url=url, num=num_changes)
                        for file in json_diff:
                            file.changes = None
                        return json_diff
            except Exception as e:
                Log.warning("could not get unified diff", cause=e)
Ejemplo n.º 26
0
 def get(self, path, **kwargs):
     url = self.settings.host + ":" + unicode(self.settings.port) + path
     try:
         response = http.get(url, **kwargs)
         if response.status_code not in [200]:
             Log.error(response.reason+": "+response.all_content)
         if self.debug:
             Log.note("response: {{response}}", response=strings.limit(utf82unicode(response.all_content), 130))
         details = wrap(convert.json2value(utf82unicode(response.all_content)))
         if details.error:
             Log.error(details.error)
         return details
     except Exception, e:
         Log.error("Problem with call to {{url}}", url=url, cause=e)
Ejemplo n.º 27
0
 def get(self, path, **kwargs):
     url = self.settings.host + ":" + unicode(self.settings.port) + path
     try:
         response = http.get(url, **kwargs)
         if response.status_code not in [200]:
             Log.error(response.reason+": "+response.all_content)
         if self.debug:
             Log.note("response: {{response}}", response=strings.limit(utf82unicode(response.all_content), 130))
         details = wrap(convert.json2value(utf82unicode(response.all_content)))
         if details.error:
             Log.error(details.error)
         return details
     except Exception, e:
         Log.error("Problem with call to {{url}}", url=url, cause=e)
Ejemplo n.º 28
0
        def inner(changeset_id):
            if self.es.cluster.version.startswith("1.7."):
                query = {
                    "query": {"filtered": {
                        "query": {"match_all": {}},
                        "filter": {"and": [
                            {"prefix": {"changeset.id": changeset_id}},
                            {"range": {"etl.timestamp": {"gt": MIN_ETL_AGE}}}
                        ]}
                    }},
                    "size": 1
                }
            else:
                query = {
                    "query": {"bool": {"must": [
                        {"prefix": {"changeset.id": changeset_id}},
                        {"range": {"etl.timestamp": {"gt": MIN_ETL_AGE}}}
                    ]}},
                    "size": 1
                }

            try:
                # ALWAYS TRY ES FIRST
                with self.es_locker:
                    response = self.es.search(query)
                    json_diff = response.hits.hits[0]._source.changeset.diff
                if json_diff:
                    return json_diff
            except Exception as e:
                pass

            url = expand_template(DIFF_URL, {"location": revision.branch.url, "rev": changeset_id})
            DEBUG and Log.note("get unified diff from {{url}}", url=url)
            try:
                response = http.get(url)
                diff = response.content.decode("utf8")
                json_diff = diff_to_json(diff)
                num_changes = _count(c for f in json_diff for c in f.changes)
                if json_diff:
                    if revision.changeset.description.startswith("merge "):
                        return None  # IGNORE THE MERGE CHANGESETS
                    elif num_changes < MAX_DIFF_SIZE:
                        return json_diff
                    else:
                        Log.warning("Revision at {{url}} has a diff with {{num}} changes, ignored", url=url, num=num_changes)
                        for file in json_diff:
                            file.changes = None
                        return json_diff
            except Exception as e:
                Log.warning("could not get unified diff from {{url}}", url=url, cause=e)
Ejemplo n.º 29
0
def _get_branches_from_hg(kwarg):
    # GET MAIN PAGE
    response = http.get(kwarg.url)
    doc = BeautifulSoup(response.all_content)

    all_repos = doc("table")[1]
    branches = UniqueIndex(["name", "locale"], fail_on_dup=False)
    for i, r in enumerate(all_repos("tr")):
        dir, name = [v.text.strip() for v in r("td")]

        b = _get_single_branch_from_hg(kwarg, name, dir.lstrip("/"))
        branches.extend(b)

    # branches.add(set_default({"name": "release-mozilla-beta"}, branches["mozilla-beta", DEFAULT_LOCALE]))
    for b in list(branches["mozilla-beta", ]):
        branches.add(set_default({"name": "release-mozilla-beta"},
                                 b))  # THIS IS THE l10n "name"
        b.url = "https://hg.mozilla.org/releases/mozilla-beta"  # THIS IS THE

    for b in list(branches["mozilla-release", ]):
        branches.add(set_default({"name": "release-mozilla-release"}, b))

    for b in list(branches["mozilla-aurora", ]):
        if b.locale == "en-US":
            continue
        branches.add(set_default({"name": "comm-aurora"}, b))
        # b.url = "https://hg.mozilla.org/releases/mozilla-aurora"

    for b in list(branches):
        if b.name.startswith("mozilla-esr"):
            branches.add(set_default({"name": "release-" + b.name},
                                     b))  # THIS IS THE l10n "name"
            b.url = "https://hg.mozilla.org/releases/" + b.name

    #CHECKS
    for b in branches:
        if b.name != b.name.lower():
            Log.error("Expecting lowercase name")
        if not b.locale:
            Log.error("Not expected")
        if not b.url.startswith("http"):
            Log.error("Expecting a valid url")
        if not b.etl.timestamp:
            Log.error("Expecting a timestamp")

    return branches
Ejemplo n.º 30
0
    def test_longest_running_tests(self):
        test = wrap({
            "query": {
                "sort": {
                    "sort": -1,
                    "field": "avg"
                },
                "from": {
                    "from":
                    "unittest",
                    "where": {
                        "and": [{
                            "gt": {
                                "build.date": "1439337600"
                            }
                        }]
                    },
                    "groupby": [
                        "build.platform", "build.type", "run.suite",
                        "result.test"
                    ],
                    "select": [{
                        "aggregate": "avg",
                        "name": "avg",
                        "value": "result.duration"
                    }],
                    "format":
                    "table",
                    "limit":
                    100
                },
                "limit": 100,
                "format": "list"
            }
        })
        query = unicode2utf8(convert.value2json(test.query))
        # EXECUTE QUERY
        with Timer("query"):
            response = http.get(self.testing.query, data=query)
            if response.status_code != 200:
                error(response)
        result = json2value(utf82unicode(response.all_content))

        Log.note("result\n{{result|indent}}", {"result": result})
Ejemplo n.º 31
0
def _get_branches_from_hg(settings):
    # GET MAIN PAGE
    response = http.get(settings.url)
    doc = BeautifulSoup(response.all_content)

    all_repos = doc("table")[1]
    branches = UniqueIndex(["name", "locale"], fail_on_dup=False)
    for i, r in enumerate(all_repos("tr")):
        dir, name = [v.text.strip() for v in r("td")]

        b = _get_single_branch_from_hg(settings, name, dir.lstrip("/"))
        branches.extend(b)

    # branches.add(set_default({"name": "release-mozilla-beta"}, branches["mozilla-beta", DEFAULT_LOCALE]))
    for b in list(branches["mozilla-beta", ]):
        branches.add(set_default({"name": "release-mozilla-beta"}, b))  # THIS IS THE l10n "name"
        b.url = "https://hg.mozilla.org/releases/mozilla-beta"          # THIS IS THE

    for b in list(branches["mozilla-release", ]):
        branches.add(set_default({"name": "release-mozilla-release"}, b))

    for b in list(branches["mozilla-aurora", ]):
        if b.locale == "en-US":
            continue
        branches.add(set_default({"name": "comm-aurora"}, b))
        # b.url = "https://hg.mozilla.org/releases/mozilla-aurora"

    for b in list(branches):
        if b.name.startswith("mozilla-esr"):
            branches.add(set_default({"name": "release-" + b.name}, b))  # THIS IS THE l10n "name"
            b.url = "https://hg.mozilla.org/releases/" + b.name

    #CHECKS
    for b in branches:
        if b.name != b.name.lower():
            Log.error("Expecting lowercase name")
        if not b.locale:
            Log.error("Not expected")
        if not b.url.startswith("http"):
            Log.error("Expecting a valid url")
        if not b.etl.timestamp:
            Log.error("Expecting a timestamp")

    return branches
Ejemplo n.º 32
0
    def test_failures_by_directory(self):
        if self.not_real_service():
            return

        test = wrap({"query": {
            "from": {
                "type": "elasticsearch",
                "settings": {
                    "host": ES_CLUSTER_LOCATION,
                    "index": "unittest",
                    "type": "test_result"
                }
            },
            "select": [
                {
                    "aggregate": "count"
                }
            ],
            "edges": [
                "result.test",
                "result.ok"
            ],
            "where": {
                "prefix": {
                    "result.test": "/"
                }
            },
            "format": "table"
        }})

        query = convert.unicode2utf8(convert.value2json(test.query))
        # EXECUTE QUERY
        with Timer("query"):
            response = http.get(self.service_url, data=query)
            if response.status_code != 200:
                error(response)
        result = convert.json2value(convert.utf82unicode(response.all_content))

        Log.note("result\n{{result|indent}}", {"result": result})
Ejemplo n.º 33
0
        def inner(changeset_id):
            if self.es.cluster.version.startswith("1.7."):
                query = {
                    "query": {"filtered": {
                        "query": {"match_all": {}},
                        "filter": {"and": [
                            {"prefix": {"changeset.id": changeset_id}},
                            {"range": {"etl.timestamp": {"gt": MIN_ETL_AGE}}}
                        ]}
                    }},
                    "size": 1
                }
            else:
                query = {
                    "query": {"bool": {"must": [
                        {"prefix": {"changeset.id": changeset_id}},
                        {"range": {"etl.timestamp": {"gt": MIN_ETL_AGE}}}
                    ]}},
                    "size": 1
                }

            try:
                # ALWAYS TRY ES FIRST
                with self.es_locker:
                    response = self.es.search(query)
                    moves = response.hits.hits[0]._source.changeset.moves
                if moves:
                    return moves
            except Exception as e:
                pass

            url = expand_template(DIFF_URL, {"location": revision.branch.url, "rev": changeset_id})
            DEBUG and Log.note("get unified diff from {{url}}", url=url)
            try:
                moves = http.get(url).content.decode('latin1')  # THE ENCODING DOES NOT MATTER BECAUSE WE ONLY USE THE '+', '-' PREFIXES IN THE DIFF
                return diff_to_moves(text_type(moves))
            except Exception as e:
                Log.warning("could not get unified diff from {{url}}", url=url, cause=e)
Ejemplo n.º 34
0
    def _open(self):
        """ DO NOT USE THIS UNLESS YOU close() FIRST"""
        if self.settings.ssl.ca.startswith("https://"):
            self.pemfile_url = self.settings.ssl.ca
            self.pemfile = File("./resources/pem") / self.settings.host
            self.pemfile.write_bytes(http.get(self.pemfile_url).content)
            self.settings.ssl.ca = self.pemfile.abspath

        try:
            self.db = connect(
                host=self.settings.host,
                port=self.settings.port,
                user=coalesce(self.settings.username, self.settings.user),
                passwd=coalesce(self.settings.password, self.settings.passwd),
                db=coalesce(self.settings.schema, self.settings.db),
                read_timeout=coalesce(self.settings.read_timeout,
                                      (EXECUTE_TIMEOUT / 1000) -
                                      10 if EXECUTE_TIMEOUT else None, 5 * 60),
                charset=u"utf8",
                use_unicode=True,
                ssl=coalesce(self.settings.ssl, None),
                cursorclass=cursors.SSCursor)
        except Exception as e:
            if self.settings.host.find("://") == -1:
                Log.error(u"Failure to connect to {{host}}:{{port}}",
                          host=self.settings.host,
                          port=self.settings.port,
                          cause=e)
            else:
                Log.error(
                    u"Failure to connect.  PROTOCOL PREFIX IS PROBABLY BAD", e)
        self.cursor = None
        self.partial_rollback = False
        self.transaction_level = 0
        self.backlog = [
        ]  # accumulate the write commands so they are sent at once
        if self.readonly:
            self.begin()
Ejemplo n.º 35
0
def get_http(ref, url):
    from pyLibrary.env import http

    params = url.query
    new_value = _convert.json2value(http.get(ref), params=params, flexible=True, leaves=True)
    return new_value
Ejemplo n.º 36
0
def test_empty_query(config, app):
    url = "http://localhost:" + text_type(config.flask.port) + "/tuid"
    response = http.get(url)
    assert response.status_code == 400
    assert response.content == EXPECTING_QUERY
Ejemplo n.º 37
0
def test_query_error(config, app):
    url = "http://localhost:" + text_type(config.flask.port) + "/tuid"
    http.get(url)
Ejemplo n.º 38
0
def test_query_too_big(config, app):
    url = "http://localhost:" + text_type(config.flask.port) + "/tuid/"
    name = "a"*10000000
    response = http.get(url, json={"from": name})
    assert response.status_code == 400
    assert response.content == b"request too large"
Ejemplo n.º 39
0
def _get_single_branch_from_hg(settings, description, dir):
    if dir == "users":
        return []
    response = http.get(settings.url + "/" + dir)
    doc = BeautifulSoup(response.all_content, "html.parser")

    output = []
    try:
        all_branches = doc("table")[0]
    except Exception:
        return []

    for i, b in enumerate(all_branches("tr")):
        if i == 0:
            continue  # IGNORE HEADER
        columns = b("td")

        try:
            path = columns[0].a.get('href')
            if path == "/":
                continue

            name, desc, last_used = [c.text.strip() for c in columns][0:3]

            if last_used.startswith('at'):
                last_used = last_used[2:]

            detail = Data(
                name=name.lower(),
                locale=DEFAULT_LOCALE,
                parent_name=description,
                url=settings.url + path,
                description=desc,
                last_used=Date(last_used),
                etl={"timestamp": Date.now()}
            )
            if detail.description == "unknown":
                detail.description = None

            # SOME BRANCHES HAVE NAME COLLISIONS, IGNORE LEAST POPULAR
            if path in [
                "/projects/dxr/",                   # moved to webtools
                "/build/compare-locales/",          # ?build team likes to clone?
                "/build/puppet/",                   # ?build team likes to clone?
                "/SeaMonkey/puppet/",               # looses the popularity contest
                "/releases/gaia-l10n/v1_2/en-US/",  # use default branch
                "/releases/gaia-l10n/v1_3/en-US/",  # use default branch
                "/releases/gaia-l10n/v1_4/en-US/",  # use default branch
                "/releases/gaia-l10n/v2_0/en-US/",  # use default branch
                "/releases/gaia-l10n/v2_1/en-US/",  # use default branch
                "/build/autoland/"
            ]:
                continue

            # MARKUP BRANCH IF LOCALE SPECIFIC
            if path.startswith("/l10n-central"):
                _path = path.strip("/").split("/")
                detail.locale = _path[-1]
                detail.name = "mozilla-central"
            elif path.startswith("/releases/l10n/"):
                _path = path.strip("/").split("/")
                detail.locale = _path[-1]
                detail.name = _path[-2].lower()
            elif path.startswith("/releases/gaia-l10n/"):
                _path = path.strip("/").split("/")
                detail.locale = _path[-1]
                detail.name = "gaia-" + _path[-2][1::]
            elif path.startswith("/weave-l10n"):
                _path = path.strip("/").split("/")
                detail.locale = _path[-1]
                detail.name = "weave"

            if BRANCH_WHITELIST is not None:
                found = False
                for br in BRANCH_WHITELIST:
                    if br in str(detail.name):
                        found = True
                        break
                if not found:
                    continue

            Log.note("Branch {{name}} {{locale}}", name=detail.name, locale=detail.locale)
            output.append(detail)
        except Exception as e:
            Log.warning("branch digestion problem", cause=e)

    return output
Ejemplo n.º 40
0
def test_shutdown(config, app):
    # SHOULD NOT ACCEPT A SHUTDOWN COMMAND, WILL BREAK OTHER TESTS
    url = "http://localhost:" + text_type(config.flask.port) + "/shutdown"
    http.get(url)
    Log.alert("/shutdown sent, should have no effect")
Ejemplo n.º 41
0
 def read_lines(self, key):
     url = self.url + "/" + key
     return http.get(url).all_lines
Ejemplo n.º 42
0
def test_query_too_big(config, app):
    url = "http://localhost:" + text_type(config.flask.port) + "/tuid/"
    name = "a" * 10000000
    response = http.get(url, json={"from": name})
    assert response.status_code == 400
    assert response.content == b"request too large"
Ejemplo n.º 43
0
def test_shutdown(config, app):
    # SHOULD NOT ACCEPT A SHUTDOWN COMMAND, WILL BREAK OTHER TESTS
    url = "http://localhost:" + text_type(config.flask.port) + "/shutdown"
    http.get(url)
    Log.alert("/shutdown sent, should have no effect")
Ejemplo n.º 44
0
        def inner(changeset_id):
            if self.es.cluster.version.startswith("1.7."):
                query = {
                    "query": {
                        "filtered": {
                            "query": {
                                "match_all": {}
                            },
                            "filter": {
                                "and": [{
                                    "prefix": {
                                        "changeset.id": changeset_id
                                    }
                                }, {
                                    "range": {
                                        "etl.timestamp": {
                                            "gt": MIN_ETL_AGE
                                        }
                                    }
                                }]
                            }
                        }
                    },
                    "size": 1
                }
            else:
                query = {
                    "query": {
                        "bool": {
                            "must": [{
                                "prefix": {
                                    "changeset.id": changeset_id
                                }
                            }, {
                                "range": {
                                    "etl.timestamp": {
                                        "gt": MIN_ETL_AGE
                                    }
                                }
                            }]
                        }
                    },
                    "size": 1
                }

            try:
                # ALWAYS TRY ES FIRST
                with self.es_locker:
                    response = self.es.search(query)
                    moves = response.hits.hits[0]._source.changeset.moves
                if moves:
                    return moves
            except Exception as e:
                pass

            url = expand_template(DIFF_URL, {
                "location": revision.branch.url,
                "rev": changeset_id
            })
            DEBUG and Log.note("get unified diff from {{url}}", url=url)
            try:
                moves = http.get(url).content.decode(
                    'latin1'
                )  # THE ENCODING DOES NOT MATTER BECAUSE WE ONLY USE THE '+', '-' PREFIXES IN THE DIFF
                return diff_to_moves(text_type(moves))
            except Exception as e:
                Log.warning("could not get unified diff from {{url}}",
                            url=url,
                            cause=e)
Ejemplo n.º 45
0
Archivo: s3.py Proyecto: rv404674/TUID
 def read_lines(self, key):
     url = self.url + "/" + key
     return http.get(url).all_lines
def process(source_key, source, dest_bucket, resources, please_stop=None):
    """
    SIMPLE CONVERT pulse_block INTO TALOS, IF ANY
    """
    etl_head_gen = EtlHeadGenerator(source_key)
    stats = Dict()
    counter = 0

    output = set()
    for i, pulse_line in enumerate(source.read_lines()):
        pulse_record = scrub_pulse_record(source_key, i, pulse_line, stats)
        if not pulse_record:
            continue

        if not pulse_record.payload.talos:
            continue

        all_talos = []
        etl_file = wrap({
            "id": counter,
            "file": pulse_record.payload.logurl,
            "timestamp": Date.now().unix,
            "source": pulse_record.etl,
            "type": "join"
        })
        with Timer("Read {{url}}", {"url": pulse_record.payload.logurl}, debug=DEBUG) as timer:
            try:
                response = http.get(pulse_record.payload.logurl)
                if response.status_code == 404:
                    Log.alarm("Talos log missing {{url}}", url=pulse_record.payload.logurl)
                    k = source_key + "." + unicode(counter)
                    try:
                        # IF IT EXISTS WE WILL ASSUME SOME PAST PROCESS TRANSFORMED THE MISSING DATA ALREADY
                        dest_bucket.get_key(k)
                        output |= {k}  # FOR DENSITY CALCULATIONS
                    except Exception:
                        _, dest_etl = etl_head_gen.next(etl_file, "talos")
                        dest_etl.error = "Talos log missing"
                        output |= dest_bucket.extend([{
                            "id": etl2key(dest_etl),
                            "value": {
                                "etl": dest_etl,
                                "pulse": pulse_record.payload
                            }
                        }])

                    continue
                all_log_lines = response.all_lines

                for log_line in all_log_lines:
                    s = log_line.find(TALOS_PREFIX)
                    if s < 0:
                        continue

                    log_line = strings.strip(log_line[s + len(TALOS_PREFIX):])
                    talos = convert.json2value(convert.utf82unicode(log_line))

                    for t in talos:
                        _, dest_etl = etl_head_gen.next(etl_file, "talos")
                        t.etl = dest_etl
                        t.pulse = pulse_record.payload
                    all_talos.extend(talos)
            except Exception, e:
                Log.error("Problem processing {{url}}", {
                    "url": pulse_record.payload.logurl
                }, e)
            finally:
Ejemplo n.º 47
0
def test_query_error(config, app):
    url = "http://localhost:" + text_type(config.flask.port) + "/tuid"
    response = http.get(url, json={"from": "files"})
    error = json2value(utf82unicode(response.content))
    assert response.status_code == 400
    assert "expecting a simple where clause with following structure" in error.template
Ejemplo n.º 48
0
 def _get_source_code_from_hg(self, revision, file_path):
     response = http.get(expand_template(FILE_URL, {"location": revision.branch.url, "rev": revision.changeset.id, "path": file_path}))
     return response.content.decode("utf8", "replace")
Ejemplo n.º 49
0
 def get_branches(self):
     response = http.get(self.settings.branches.url, timeout=coalesce(self.settings.timeout, 30))
     branches = convert.json2value(convert.utf82unicode(response.all_content))
     return wrap({branch.name: unwrap(branch) for branch in branches})
Ejemplo n.º 50
0
def get_ssl_pem_file(url):
    filename = File(".pem") / URL(url).host
    filename.write_bytes(http.get(url).content)
    return {"cafile": filename.abspath}
Ejemplo n.º 51
0
def test_empty_query(config, app):
    url = "http://localhost:" + text_type(config.flask.port) + "/tuid"
    response = http.get(url)
    assert response.status_code == 400
    assert response.content == b"expecting query"
Ejemplo n.º 52
0
def _get_single_branch_from_hg(settings, description, dir):
    if dir == "users":
        return []
    response = http.get(settings.url + "/" + dir)
    doc = BeautifulSoup(response.all_content, "html.parser")

    output = []
    try:
        all_branches = doc("table")[0]
    except Exception:
        return []

    for i, b in enumerate(all_branches("tr")):
        if i == 0:
            continue  # IGNORE HEADER
        columns = b("td")

        try:
            path = columns[0].a.get('href')
            if path == "/":
                continue

            name, desc, last_used = [c.text.strip() for c in columns][0:3]

            if last_used.startswith('at'):
                last_used = last_used[2:]

            detail = Data(
                name=name.lower(),
                locale=DEFAULT_LOCALE,
                parent_name=description,
                url=settings.url + path,
                description=desc,
                last_used=Date(last_used),
                etl={"timestamp": Date.now()}
            )
            if detail.description == "unknown":
                detail.description = None

            # SOME BRANCHES HAVE NAME COLLISIONS, IGNORE LEAST POPULAR
            if path in [
                "/projects/dxr/",                   # moved to webtools
                "/build/compare-locales/",          # ?build team likes to clone?
                "/build/puppet/",                   # ?build team likes to clone?
                "/SeaMonkey/puppet/",               # looses the popularity contest
                "/releases/gaia-l10n/v1_2/en-US/",  # use default branch
                "/releases/gaia-l10n/v1_3/en-US/",  # use default branch
                "/releases/gaia-l10n/v1_4/en-US/",  # use default branch
                "/releases/gaia-l10n/v2_0/en-US/",  # use default branch
                "/releases/gaia-l10n/v2_1/en-US/",  # use default branch
                "/build/autoland/"
            ]:
                continue

            # MARKUP BRANCH IF LOCALE SPECIFIC
            if path.startswith("/l10n-central"):
                _path = path.strip("/").split("/")
                detail.locale = _path[-1]
                detail.name = "mozilla-central"
            elif path.startswith("/releases/l10n/"):
                _path = path.strip("/").split("/")
                detail.locale = _path[-1]
                detail.name = _path[-2].lower()
            elif path.startswith("/releases/gaia-l10n/"):
                _path = path.strip("/").split("/")
                detail.locale = _path[-1]
                detail.name = "gaia-" + _path[-2][1::]
            elif path.startswith("/weave-l10n"):
                _path = path.strip("/").split("/")
                detail.locale = _path[-1]
                detail.name = "weave"

            Log.note("Branch {{name}} {{locale}}", name=detail.name, locale=detail.locale)
            output.append(detail)
        except Exception as e:
            Log.warning("branch digestion problem", cause=e)

    return output
Ejemplo n.º 53
0
def test_query_error(config, app):
    url = "http://localhost:" + text_type(config.flask.port) + "/tuid"
    http.get(url)