コード例 #1
0
ファイル: NLTKIndexer.py プロジェクト: ulikoehler/Translatron
    def printTokenFrequency(self):
        "Print the result generated by computeTokenFrequency in a human-readable manner"
        print(black("Statistics for document index", bold=True))
        self._printFrequencyMap(self.computeTokenFrequency(self.rwDB.iterateDocumentIndex))

        print(black("\nStatistics for entity index", bold=True))
        self._printFrequencyMap(self.computeTokenFrequency(self.rwDB.iterateEntityIndex)) 
コード例 #2
0
def importRulesForLanguage(lang, basedir="."):
    """Import ruleset from the language-specific python file"""
    moduleName = "rules.{0}".format(lang)
    print(black("Reading rules from {0}".format(moduleName), bold=True))
    langModule = importlib.import_module(moduleName)
    print(black("Found {0} rules for language {1} ({2} in compatibility mode)".format(len(langModule.rules), lang, reCompiler.numCompatRegex), bold=True))
    return langModule.rules, langModule.rule_errors
コード例 #3
0
    def printTokenFrequency(self):
        "Print the result generated by computeTokenFrequency in a human-readable manner"
        print(black("Statistics for document index", bold=True))
        self._printFrequencyMap(
            self.computeTokenFrequency(self.rwDB.iterateDocumentIndex))

        print(black("\nStatistics for entity index", bold=True))
        self._printFrequencyMap(
            self.computeTokenFrequency(self.rwDB.iterateEntityIndex))
コード例 #4
0
ファイル: pcbcheck.py プロジェクト: huigao80/PCBCheck
def checkBoardOutline(self, filepath):
    filename = os.path.basename(filepath)
    #Basic gerber checks
    checkGerberFile(self, filepath)
    #Compute board outline
    millLines = readFileLines(filepath)
    # Find factors to get absolute coordinates:
    x_factor, y_factor = findCoordinateFormat(millLines)
    #We can only interpret the file if coordinates are absolute
    if not "G90*" in millLines:
        print (yellow("Mill coordinates in %s don't seem to be absolute (G90 missing!)" % filename))
        return
    #Determine coordinate units
    unit = parseGerberUnit(millLines)
    if unit is None: #Neither inch nor mm found
        print (yellow("Could not find coordinate units (mm/in) in %s" % filename))
        return
    #Parse the aperture list
    apertures = parseGerberApertures(millLines)
    selectApertureRegex = re.compile(r"(D\d+)\*")
    move2DRegex = re.compile(r"X(\d+)Y(\d+)D(\d)\*") #Move (D2) or draw (D1)
    move1DRegex = re.compile(r"([XY])(\d+)D(\d)\*") #With only one coordinate
    #Try to interpret gerber file
    minCoords = (sys.maxsize, sys.maxsize)
    maxCoords = (0, 0)
    lastCoords = (0, 0)
    currentAperture = None
    for line in millLines:
        if selectApertureRegex.match(line):
            apertureCode = selectApertureRegex.match(line).group(1)
            currentAperture = findAperture(apertures, apertureCode)
        elif move2DRegex.match(line):
            match = move2DRegex.match(line)
            x = int(match.group(1)) / x_factor
            y = int(match.group(2)) / y_factor
        elif move1DRegex.match(line):
            match = move1DRegex.match(line)
            if match.group(1) == "X":
                x = int(match.group(2)) / x_factor
                y = lastCoords[1]
            elif match.group(1) == "Y":
                x = lastCoords[0]
                y = int(match.group(2)) / y_factor
            else: raise Exception("Internal error: Invalid coordinate type in 1D move: %s" % match.group(1))
        else: continue
        #Compute min/max coordinates
        lastCoords = (x, y)
        minCoords = (min(minCoords[0], lastCoords[0]), min(minCoords[1], lastCoords[1]))
        maxCoords = (max(maxCoords[0], lastCoords[0]), max(maxCoords[1], lastCoords[1]))
    #Compute board size (minimum enclosing rectangle)
    boardSize = (maxCoords[0] - minCoords[0], maxCoords[1] - minCoords[1])
    #Print info
    print (black("\tGerber offset: ({1:.2f} {0}, {2:.2f} {0})".format(unit, minCoords[0], minCoords[1])))
    print (black("\tBoard size (minimum rectangle): %.1f %s x %.1f %s" % \
            (boardSize[0], unit, boardSize[1], unit)))
コード例 #5
0
def importRulesForLanguage(lang, basedir="."):
    """Import ruleset from the language-specific python file"""
    moduleName = "rules.{}".format(lang)
    print(black("Reading rules from {}".format(moduleName), bold=True))
    langModule = importlib.import_module(moduleName)
    print(
        black(
            "Found {} rules for language {} ({} in compatibility mode)".format(
                len(langModule.rules), lang, reCompiler.numCompatRegex),
            bold=True))
    return langModule.rules, langModule.rule_errors
コード例 #6
0
def interval_to_string(interval):
    """
    Returns a string representation of an interval.
        :param interval: is the interval to evaluate.
    """
    # turn numbers into a single string
    nums = str(interval.start) + " " + str(interval.duration) + " " + str(
        interval.amplitude)

    # put brackets on the ends
    bracket_nums = black("[", bold=True) + nums + black("]", bold=True)

    return bracket_nums
コード例 #7
0
def fetchVideoMap(pool, lang):
    "Deprecated: Fetch individual video dubs"
    print(black("Fetching dubbed video list for lang {0}".format(lang), bold=True))
    dubbedVideoIDs = findDubbedVideos(lang)

    print(black("Fetching {0} dubbed video URLs for lang {1}"
           .format(len(dubbedVideoIDs), lang), bold=True))
    fn = functools.partial(getTranslatedVideoId, lang=lang)
    videoURLs = pool.map(fn, dubbedVideoIDs)
    
    # Remap
    return {videoId: url
             for videoId, url in zip(dubbedVideoIDs, videoURLs)
             if url is not None}
コード例 #8
0
def export_lang_to_db(lang, filt):
    count = 0
    for file in findXLIFFFiles("cache/{}".format(lang), filt=filt):
        # e.g. '1_high_priority_platform/about.donate.xliff'
        canonicalFilename = "/".join(file.split("/")[2:])
        section = canonicalFilename.partition("/")[0]
        # Dont index drafts
        if "learn.draft.xliff" in canonicalFilename:
            print(green("Skipping {}".format(canonicalFilename), bold=True))
        # relevant_for_live
        relevant_for_live = False
        if canonicalFilename in relevant_for_live_files:
            relevant_for_live = True

        print(black(file, bold=True))
        soup = parse_xliff_file(file)
        for entry in process_xliff_soup(soup, also_approved=True):
            obj = {
                "id": int(entry.ID),
                "source": entry.Source,
                "target": entry.Translated,
                "source_length": len(entry.Source),
                "is_translated": entry.IsTranslated,
                "is_approved": entry.IsApproved,
                "translation_source": "Crowdin",
                "file": canonicalFilename,
                "fileid": entry.FileID,
                "relevant_for_live": relevant_for_live
            }
            # Async write
            executor.submit(write_entry, obj, lang)
            # Stats
            count += 1
            if count % 1000 == 0:
                print("Processed {} records".format(count))
コード例 #9
0
def print_timeline(timeline, steps):
    """
    Prints a graphical representation of the timeline to the console.
        :param timeline: is the 2D array of intervals to read.
    """

    print()  # newline

    # count number of digits of left column to align numbers
    pad_length = len(str(len(timeline)))

    # print the row
    for channel in range(0, len(timeline)):
        current_index = 0
        amp = 0
        print(str(channel).zfill(pad_length), " >  ", sep='', end='')
        # print the columns
        for current_time in range(0, steps):
            (amp, current_index) = read_channel(timeline, channel,
                                                current_index, current_time)
            # make the zeros stand out. easier to look at that way
            if amp == 0:
                print(black(str(amp), bold=True), " ", sep='', end='')
            else:
                print(amp, " ", sep='', end='')
        print()  # newline
    print()  # newline
コード例 #10
0
def updateLintFromGoogleGroups(lang="de"):
    url = getLatestLintDownloadLink()
    response = requests.get(url)
    filename = os.path.join("cache", "{0}-lint.csv".format(lang))
    with open(filename, "w") as outfile:
        outfile.write(response.text)
    print(black("Updated %s" % filename, bold=True))
コード例 #11
0
def loadTranslations(conn, recordTable=1, indexTable=2):
    """
    Loads all PO strings in the database and builds a Polyglott Index
    """
    print(black("Deleting old tables...", bold=True))
    # Delete old tables (implicitly closes tables)
    conn.deleteRange(recordTable, startKey=None, endKey=None)
    conn.deleteRange(indexTable, startKey=None, endKey=None)

    print(black("Deleting old tables...", bold=True))
    # Table 1 stores msgid => NUL-separated list of records
    #   Record: Langcode (KA-style) + ASCII record separator (0x1D) + msgstr
    conn.openTable(recordTable, mergeOperator="NULAPPEND")

    # Table 2 stores key => msgid
    #   where "key" is any msgid or msgstr.
    #   The appropriate values can be looked up in table 1 using the msgid key
    conn.openTable(indexTable, mergeOperator="REPLACE")

    for lang, langpath in findAvailableLanguages().items():
        print(black("Reading PO files for language {}".format(lang,
                                                              bold=True)))
        for filename in findPOFiles(langpath):
            # NOTE: This loop writes one value dict per file per file to
            #  avoid tons of Python function calls.
            # The large values are be handled in C++ code efficiently.
            print("\tProcessing {}".format(filename))
            po = polib.pofile(filename)
            # Write table 1
            values = {
                entry.msgid: lang + "\x1D" + entry.msgstr
                for entry in po if entry.msgstr.strip()
            }
            conn.put(recordTable, values)
            # Write table 2 (index)
            values = {
                entry.msgstr: entry.msgid
                for entry in po if entry.msgstr.strip()
            }
            values2 = {entry.msgid: entry.msgid for entry in po}
            conn.put(indexTable, values)
            conn.put(indexTable, values2)
    # Perform anticipatory compation on both tables
    print(black("Compacting language table...", bold=True))
    conn.compact(recordTable)
    print(black("Compacting index table...", bold=True))
    conn.compact(indexTable)
コード例 #12
0
def buildPolyglottIndex(args):
    import YakDB
    print(black("Connecting to YakDB...", bold=True))

    conn = YakDB.Connection()
    conn.connect("tcp://localhost:7100")

    loadTranslations(conn, args.table, args.table + 1)
コード例 #13
0
def updateTranslationFilemapCache(lang="de"):
    """Re-download the translation filemap cache"""
    print(black("Updating translation filemap for {0}".format(lang), bold=True))
    filename = translationFilemapCacheFilename(lang)
    with open(filename, "w") as outfile:
        translation_filemap = downloadTranslationFilemap(lang)
        json.dump(translation_filemap, outfile)
    return translation_filemap
コード例 #14
0
def buildPolyglottIndex(args):
    import YakDB
    print(black("Connecting to YakDB...", bold=True))

    conn = YakDB.Connection()
    conn.connect("tcp://localhost:7100")

    loadTranslations(conn, args.table, args.table + 1)
コード例 #15
0
def findAllLanguages():
    """
    Acquire a dictionary language -> lang ID from Crowdin
    """
    "Find a list of Crowdin language codes to which KA is translated to"
    # Acquire language list
    print(black("Fetching language list...", bold=True))
    response = requests.get("https://crowdin.com/project/khanacademy")
    txt = response.text
    langs = re.findall(r"https?://[a-z0-9]*\.cloudfront\.net/images/flags/([^\.]+)\.png", txt)
    print(black("Fetching language IDs...", bold=True))
    # English is the source language
    if "en-US" in langs:
        langs.remove("en-US")
    # Fetch lang IDs in parallel
    pool = Pool(32)
    return dict(zip(langs, pool.map(fetchLanguageID, langs)))
コード例 #16
0
def updateTranslationFilemapCache(lang="de"):
    """Re-download the translation filemap cache"""
    print(black("Updating translation filemap for {0}".format(lang),
                bold=True))
    filename = translationFilemapCacheFilename(lang)
    with open(filename, "w") as outfile:
        translation_filemap = downloadTranslationFilemap(lang)
        json.dump(translation_filemap, outfile)
        return translation_filemap
コード例 #17
0
def fetchVideoMap(pool, lang):
    "Deprecated: Fetch individual video dubs"
    print(
        black("Fetching dubbed video list for lang {0}".format(lang),
              bold=True))
    dubbedVideoIDs = findDubbedVideos(lang)

    print(
        black("Fetching {0} dubbed video URLs for lang {1}".format(
            len(dubbedVideoIDs), lang),
              bold=True))
    fn = functools.partial(getTranslatedVideoId, lang=lang)
    videoURLs = pool.map(fn, dubbedVideoIDs)

    # Remap
    return {
        videoId: url
        for videoId, url in zip(dubbedVideoIDs, videoURLs) if url is not None
    }
コード例 #18
0
def updateVideoMap(args):
    """
    Generates VideoTranslations.json from Khan Academy data
    """
    pool = Pool(48)
    languages = list(sorted(list(findAllLanguages())))

    print(black("Fetching language videomaps", bold=True))
    langresults = pool.map(fetchVideoTranslationsCSV, languages)

    videoMap = defaultdict(dict)
    for language, langresult in zip(languages, langresults):
        # Insert results into main language map
        for slug, url, orig_url in langresult:
            videoMap[slug][language] = url
            videoMap[slug]["en"] = orig_url

    print(black("Writing JSON videomap...", bold=True))
    with open(os.path.join("cache", "VideoMap.json"), "w") as outfile:
        json.dump(videoMap, outfile)
コード例 #19
0
def updateVideoMap(args):
    """
    Generates VideoTranslations.json from Khan Academy data
    """
    pool = Pool(48)
    languages = list(sorted(list(findAllLanguages())))

    print(black("Fetching language videomaps", bold=True))
    langresults = pool.map(fetchVideoTranslationsCSV, languages)

    videoMap = defaultdict(dict)
    for language, langresult in zip(languages, langresults):
        # Insert results into main language map
        for slug, url, orig_url in langresult:
            videoMap[slug][language] = url
            videoMap[slug]["en"] = orig_url

    print(black("Writing JSON videomap...", bold=True))
    with open(os.path.join("cache", "VideoMap.json"), "w") as outfile:
        json.dump(videoMap, outfile)
コード例 #20
0
def loadTranslations(conn, recordTable=1, indexTable=2):
    """
    Loads all PO strings in the database and builds a Polyglott Index
    """
    print(black("Deleting old tables...", bold=True))
    # Delete old tables (implicitly closes tables)
    conn.deleteRange(recordTable, startKey=None, endKey=None)
    conn.deleteRange(indexTable, startKey=None, endKey=None)

    print(black("Deleting old tables...", bold=True))
    # Table 1 stores msgid => NUL-separated list of records
    #   Record: Langcode (KA-style) + ASCII record separator (0x1D) + msgstr
    conn.openTable(recordTable, mergeOperator="NULAPPEND")

    # Table 2 stores key => msgid
    #   where "key" is any msgid or msgstr.
    #   The appropriate values can be looked up in table 1 using the msgid key
    conn.openTable(indexTable, mergeOperator="REPLACE")

    for lang, langpath in findAvailableLanguages().items():
        print(black("Reading PO files for language {}".format(lang, bold=True)))
        for filename in findPOFiles(langpath):
            # NOTE: This loop writes one value dict per file per file to
            #  avoid tons of Python function calls.
            # The large values are be handled in C++ code efficiently.
            print("\tProcessing {}".format(filename))
            po = polib.pofile(filename)
            # Write table 1
            values = {entry.msgid: lang + "\x1D" + entry.msgstr
                      for entry in po if entry.msgstr.strip()}
            conn.put(recordTable, values)
            # Write table 2 (index)
            values = {entry.msgstr: entry.msgid for entry in po if entry.msgstr.strip()}
            values2 = {entry.msgid: entry.msgid for entry in po}
            conn.put(indexTable, values)
            conn.put(indexTable, values2)
    # Perform anticipatory compation on both tables
    print(black("Compacting language table...", bold=True))
    conn.compact(recordTable)
    print(black("Compacting index table...", bold=True))
    conn.compact(indexTable)
コード例 #21
0
def performRender(args):
    # Download / update if requested
    if args.download:
        download()

    # Create directory
    if not args.outdir:
        args.outdir = "output-{0}".format(args.language)
    if not os.path.isdir(args.outdir):
        os.mkdir(args.outdir)

    renderer = HTMLHitRenderer(args.outdir, args.language)

    # Generate HTML
    if not args.no_lint:
        print(black("Rendering lint...", bold=True))
        success = False
        for i in range(25):
            try:
                renderer.renderLintHTML()
                success = True
                break
            except NoResultException:
                print(red("Lint fetch error, retrying..."))
        if not success:
            print(red("Lint fetch error (retries exhausted)", bold=True))


    if not args.only_lint:
        # Import
        potDir = os.path.join("cache", args.language)
        print(black("Reading files from {0} folder...".format(potDir), bold=True))
        poFiles = readPOFiles(potDir)
        print(black("Read {0} files".format(len(poFiles)), bold=True))
        # Compute hits
        print(black("Computing rules...", bold=True))
        renderer.computeRuleHitsForFileSet(poFiles)
        # Ensure the HUGE po stuff goes out of scope ASAP
        poFiles = None

        # Generate HTML
        print(black("Rendering HTML...", bold=True))
        renderer.hitsToHTML()

        # Generate filestats.json
        print (black("Generating JSON API files...", bold=True))
        renderer.writeStatsJSON()

    # If data is present, generate subtitle information
    videosJSONPath = os.path.join("cache", "videos.json")
    if os.path.isfile(videosJSONPath):
        print (black("Rendering subtitles overview...", bold=True))
        with open(videosJSONPath) as infile:
            exercises = json.load(infile)
        subtitleTemplate = renderer.env.get_template("subtitles.html")
        writeToFile(os.path.join(args.outdir, "subtitles.html"), subtitleTemplate.render(exercises=exercises))
コード例 #22
0
def performRender(args):
    # Download / update if requested
    if args.download:
        download()

    # Create directory
    if not args.outdir:
        args.outdir = "output"
    os.makedirs(args.outdir, exist_ok=True)

    renderer = JSONHitRenderer(args.outdir, args.language, args.num_processes)

    # Import
    potDir = os.path.join("cache", args.language)
    xliffFiles = findXLIFFFiles(potDir, filt=args.filter)
    print(
        black("Reading {} files from {} folder...".format(
            len(xliffFiles), potDir),
              bold=True))
    # Compute hits
    print(black("Computing rules...", bold=True))
    renderer.computeRuleHitsForFileSet(xliffFiles)

    # Generate HTML
    print(black("Rendering HTML...", bold=True))
    renderer.exportHitsAsJSON()

    # Generate filestats.json
    print(black("Generating JSON API files...", bold=True))
    renderer.writeStatsJSON()

    # If data is present, generate subtitle information
    videosJSONPath = os.path.join("cache", "videos.json")
    if os.path.isfile(videosJSONPath):
        print(black("Rendering subtitles overview...", bold=True))
        with open(videosJSONPath) as infile:
            exercises = json.load(infile)
        subtitleTemplate = renderer.env.get_template("subtitles.html")
        writeToFile(os.path.join(args.outdir, "subtitles.html"),
                    subtitleTemplate.render(exercises=exercises))
コード例 #23
0
def ensureFileIsPresent(asset, directory, forceDownload=False):
    (filename, url) = asset
    filepath = os.path.join(directory, filename)
    if url is None:  # --> no need to download
        return
    if not os.path.isfile(filepath) or forceDownload:
        #Create directory if required
        dirname = os.path.dirname(filepath)
        if not os.path.exists(dirname):
            os.makedirs(dirname)
        #Perform download
        print(black("Downloading %s" % filename, bold=True))
        urllib.request.urlretrieve(url, filepath)
コード例 #24
0
def autobuild_python_package(nameorurl, suffix, version=None, depends=[], build_depends=[], py2=True, py3=True, remove_pyc=False):
    """
    Automatically build a python package.
    Takes either a git+http(s) or git:// URL or a package name.
    In case of a package name, the package is downloaded from PyPI
    """

    if nameorurl.startswith(("git+https://", "git+http://", "git://")) or \
       (nameorurl.startswith(("http://", "https://")) and nameorurl.endswith(".git")):
        # Git package download
        pkgname = nameorurl.rpartition("/")[2].rstrip(".git")
        print(black("Cloning {} {}...".format(pkgname,
            "(branch {})".format(version) if version else ""), bold=True))
        set_name(pkgname.lower())
        git_clone(nameorurl, branch=version)

    else: # Normal package download
        set_name(nameorurl)
        remove_old_buildtree()
        print(black("Fetching latest PyPI revision of {}".format(nameorurl), bold=True))
        pkgversion, url = find_latest_pypi_version(get_name(), forced_version=version)

        # Download and extract archive
        print(black("Downloading & extracting {}-{}".format(
            nameorurl, pkgversion), bold=True))
        wget_download(url)

    # Remove pyc (workaround) if enabled
    if remove_pyc:
        cmd("find . -name '*.pyc' -print0 | xargs -0 rm")

    # Perform build of deb package using stdeb
    print(black("Building deb package", bold=True))
    build_stdeb(suffix, py2, py3, depends=depends, build_depends=build_depends)

    # Move deb packages to current directory
    print(black("Moving build result", bold=True))
    cmd("find deb_dist -maxdepth 1 -type f -exec mv {} .. \;")
コード例 #25
0
    def computeRuleHits(self, po, filename="[unknown filename]"):
        """
        Compute all rule hits for a single parsed PO file and return a list of futures
        that return (filename, rule, results tuples).

        po must be a function that returns a pofile object
        """
        def _wrapper(rule, po, filename):
            # print("   {} => Rule {}".format(filename, rule))
            return (filename, rule, list(rule.apply_to_po(po, filename=filename)))
        # Actually read PO file
        print(black("Reading {} ...".format(filename), bold=True))
        po = po()
        futures = [self.executor.submit(_wrapper, rule, po, filename) for rule in self.rules]
        return futures
コード例 #26
0
def delayedIndexPattern(lang, pattern, delay=15):
    if pattern in currently_indexing:
        print(black("Ignoring index request for '{}', already in queue".format(pattern), bold=True))
        return
    # Avoid duplicate indexing
    currently_indexing.append(pattern)
    # Delay
    time.sleep(delay) # Allow DB to sync
    # Index with both relevant_for_live settings
    executor.submit(index_pattern, client, lang, pattern, True)
    executor.submit(index_pattern, client, lang, pattern, False)
    # Remove from queue
    try:
        currently_indexing.remove(pattern)
    except ValueError:
        pass
コード例 #27
0
def export_lang_to_db(lang, filt):
    count = 0
    indexer = TextTagIndexer(lang)
    for file in findXLIFFFiles("cache/{}".format(lang), filt=filt):
        # e.g. '1_high_priority_platform/about.donate.xliff'
        canonicalFilename = "/".join(file.split("/")[2:])
        print(black(file, bold=True))
        soup = parse_xliff_file(file)
        for entry in process_xliff_soup(soup, also_approved=True):
            indexer.add(entry.Source, entry.Translated,
                        file.rpartition("/")[2], entry.IsApproved)
            # Stats
            count += 1
            if count % 1000 == 0:
                print("Processed {} records".format(count))
    return ttt(lang, list(indexer._convert_to_json()))
コード例 #28
0
def renderLint(outdir, kalangcode):
    "Parse & render lint"
    # Map from KA code to crowdin code
    langMap = {"pt": "pt-BR", "pt-pt": "pt-PT", "ur": "ur-PK", "es": "es-ES", "hy": "hy-AM"}
    lang = langMap[kalangcode] if kalangcode in langMap else kalangcode
    print(black("Rendering lint for {0} ({1})".format(lang, kalangcode), bold=True))
    # Generate lint report
    lintFilename = os.path.join("cache", "{0}-lint.csv".format(kalangcode))
    if os.path.isfile(lintFilename):
        lintEntries = list(readAndMapLintEntries(lintFilename, lang))
        # Write JSON
        jsonEntries = list(map(operator.methodcaller("_asdict"), lintEntries))
        os.makedirs(os.path.join(outdir, lang), exist_ok=True)
        writeJSONToFile(os.path.join(outdir, lang, "lint.json"), jsonEntries)
    else:
        print("Skipping lint ({0} does not exist)".format(lintFilename))
コード例 #29
0
def _translate(entry, translator, force=False, tries_left=5):
    engl = entry["english"]
    if not force and entry["translated"] is not None and entry["translated"] == "":
        return entry # leave as is
    try:
        transl = translator[tries_left].translate(engl)
        if transl is None:
            if tries_left > 0:
                print("Retrying '{}'".format(engl))
                return _translate(entry, translator, force, tries_left - 1)
            else:
                print(red("Wont retry '{}'".format(engl)))
    except:
        print(black("Autotranslate fail for string '{}'".format(engl), bold=True))
        traceback.print_exception(*sys.exc_info())
        return entry
    entry["translated"] = transl
    #print("{} ==> {}".format(engl, entry["translated"]))
    return entry
コード例 #30
0
ファイル: pcbcheck.py プロジェクト: huigao80/PCBCheck
def checkExcellonMetric(self, filepath):
    "Check if a given file is a metric excellon file"
    filename = os.path.basename(filepath)
    lines = readFileLines(filepath)
    #Check for excellon header
    if lines[0] != "M48":
        print red("Can't find Excellon drill header (M48) in %s" % filename, bold="True")
    #Check for metric dimension: Line like METRIC,0000.00
    if lines[1].partition(",")[0] != "METRIC":
        print red("Excellon drill program %s does not seem to be metric" % filename, bold="True")
    #
    # Drill statistics
    #
    toolStats = extractToolStatistics(lines)
    print(black(self.name + ":", bold=True))
    for diameter, numDrills in toolStats.iteritems():
        print("\t%d through holes of diameter %.2fmm" % (numDrills, diameter))
    #Print "None" if there are no holes in this file
    if not toolStats:
        print "\tNone"
コード例 #31
0
def checkExcellonMetric(self, filepath):
    "Check if a given file is a metric excellon file"
    filename = os.path.basename(filepath)
    lines = readFileLines(filepath)
    #Check for excellon header
    if lines[0] != "M48":
        print(red("Can't find Excellon drill header (M48) in %s" % filename, bold="True"))
    #Check for metric dimension: Line like METRIC,0000.00
    if lines[1].partition(",")[0] != "METRIC":
        print(red("Excellon drill program %s does not seem to be metric" % filename, bold="True"))
    #
    # Drill statistics
    #
    toolStats = extractToolStatistics(lines)
    print(black(self.name + ":", bold=True))
    for diameter, numDrills in toolStats.items():
        print("\t%d through holes of diameter %.2fmm" % (numDrills, diameter))
    #Print "None" if there are no holes in this file
    if not toolStats:
        print("\tNone")
コード例 #32
0
ファイル: parse.py プロジェクト: scchow/PneuNet
def parse_check_format(intv, verbose=False):
    """
    Checks if a string could contain an interval. This function just exists to
    shorten read_timeline() to something that Python won't throw warnings about.
    There's no need to call this outside that function.
        :param intv: is the string to check.
        :param verbose: is an optional flag.
            When true, extra parsing information is printed to the console. Defaults to false.
    """
    # if the string is shorter than 5 characters, it can't be [#] [#] [#], since that's
    # 3 numbers + 2 spaces. if it is less than 5, it's not an interval
    if len(intv) < 5:
        if verbose:
            print(black("\tinvalid length > ", bold=True), end='')
            print(add_quotes(intv))
        return False

    # haven't ruled it out yet
    if verbose:
        print(blue("\tpossible interval > ", bold=True), end='')
        print(add_quotes(intv))
    return True
コード例 #33
0
def renderLint(outdir, kalangcode):
    "Parse & render lint"
    # Map from KA code to crowdin code
    langMap = {
        "pt": "pt-BR",
        "pt-pt": "pt-PT",
        "ur": "ur-PK",
        "es": "es-ES",
        "hy": "hy-AM"
    }
    lang = langMap[kalangcode] if kalangcode in langMap else kalangcode
    print(
        black("Rendering lint for {0} ({1})".format(lang, kalangcode),
              bold=True))
    # Generate lint report
    lintFilename = os.path.join("cache", "{0}-lint.csv".format(kalangcode))
    if os.path.isfile(lintFilename):
        lintEntries = list(readAndMapLintEntries(lintFilename, lang))
        # Write JSON
        jsonEntries = list(map(operator.methodcaller("_asdict"), lintEntries))
        os.makedirs(os.path.join(outdir, lang), exist_ok=True)
        writeJSONToFile(os.path.join(outdir, lang, "lint.json"), jsonEntries)
    else:
        print("Skipping lint ({0} does not exist)".format(lintFilename))
コード例 #34
0
def performRender(args):
    # Download / update if requested
    if args.download:
        download()

    # Create directory
    if not args.outdir:
        args.outdir = "output"
    os.makedirs(args.outdir, exist_ok=True)

    renderer = JSONHitRenderer(args.outdir, args.language)

    # Import
    potDir = os.path.join("cache", args.language)
    print(black("Reading files from {0} folder...".format(potDir), bold=True))
    poFiles = readPOFiles(potDir)
    print(black("Read {0} files".format(len(poFiles)), bold=True))
    # Compute hits
    print(black("Computing rules...", bold=True))
    renderer.computeRuleHitsForFileSet(poFiles)
    # Ensure the HUGE po stuff goes out of scope ASAP
    del poFiles

    # Generate HTML
    print(black("Rendering HTML...", bold=True))
    renderer.exportHitsAsJSON()

    # Generate filestats.json
    print (black("Generating JSON API files...", bold=True))
    renderer.writeStatsJSON()

    # If data is present, generate subtitle information
    videosJSONPath = os.path.join("cache", "videos.json")
    if os.path.isfile(videosJSONPath):
        print (black("Rendering subtitles overview...", bold=True))
        with open(videosJSONPath) as infile:
            exercises = json.load(infile)
        subtitleTemplate = renderer.env.get_template("subtitles.html")
        writeToFile(os.path.join(args.outdir, "subtitles.html"), subtitleTemplate.render(exercises=exercises))
コード例 #35
0
def add_quotes(message):
    """
    Adds grey quotes around text.
        :param message: is the string to put quotes around.
    """
    return black("\"", bold=True) + str(message) + black("\"", bold=True)
コード例 #36
0
        try:
            slug, orig_vid, vid = row[8], row[5], row[6]
            # Ignore non translated videos
            if not vid: continue
            url_tpl = "https://www.youtube.com/watch?v={0}"
            url = url_tpl.format(vid)
            orig_url = url_tpl.format(orig_vid)
            result.append((slug, url, orig_url))
        except IndexError:
            continue
    return result

if __name__ == "__main__":
    pool = Pool(48)
    languages = list(sorted(list(findAllLanguages())))

    print(black("Fetching language videomaps", bold=True))
    langresults = pool.map(fetchVideoTranslationsCSV, languages)

    videoMap = defaultdict(dict)
    for language, langresult in zip(languages, langresults):
        # Insert results into main language map
        for slug, url, orig_url in langresult:
            videoMap[slug][language] = url
            videoMap[slug]["en"] = orig_url

    print(black("Writing JSON videomap...", bold=True))
    with open("VideoMap.json", "w") as outfile:
        json.dump(videoMap, outfile)
    
コード例 #37
0
ファイル: pcbcheck.py プロジェクト: huigao80/PCBCheck
    ExpectedFile(".top", "Top copper layer", "RS-274X", checkCopperLayer),
    ExpectedFile(".bot", "Bottom copper layer", "RS-274X", checkCopperLayer),
    ExpectedFile(".smt", "Solder mask top", "RS-274X", checkGerberFile),
    ExpectedFile(".smb", "Solder mask bottom", "RS-274X", checkGerberFile),
    ExpectedFile(".plt", "Silk screen top", "RS-274X", checkGerberFile),
    ExpectedFile(".mil", "Board outline", "RS-274X", checkBoardOutline),
    #Drilling
    ExpectedFile(".pth", "Plated through holes", "Excellon", checkExcellonMetric),
    ExpectedFile(".npth", "Non-plated through holes", "Excellon", checkExcellonMetric),
]

if __name__ == "__main__":
    #Parse commandline arguments
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument("directory", help="The directory to scan for project Gerber file")
    parser.add_argument("--gerbv", action="store_true", help="Run gerbv on the files")
    args = parser.parse_args()
    #Perform check
    files = os.listdir(args.directory)
    projectName = extractProjectPrefix(files)
    print black("Project name: %s" % projectName)
    checkedFiles = [checkFile(args.directory, f, projectName) for f in expectedFiles]
    unknownFiles = set(files) - set(checkedFiles)
    if unknownFiles:
        print red("Found unknown files: %s" % ",".join(unknownFiles))
    #Open viewer if enabled
    if args.gerbv:
        filePaths = [os.path.join(args.directory, f) for  f in files]
        subprocess.call(["gerbv"] + filePaths)
コード例 #38
0
ファイル: parse.py プロジェクト: scchow/PneuNet
def read_timeline(filename, verbose=False):
    """
    Reads a timeline of intervals from a text file. Returns the timeline and if there are errors.
        :param filename: is the path to the timeline file.
        :param verbose: is an optional flag.
            When true, extra parsing information is printed to the console. Defaults to false.
    """
    # make timeline array
    timeline = []
    steps = 0

    count, total = 0, 0

    # Yes, this is the second part where this is checked. Gotta be sure.
    if not file_exists(filename):
        if verbose:
            print(add_quotes(filename), "is not a file")
        return False

    errors = False

    with open(filename) as lines:
        for num, line in enumerate(lines):
            line = line.strip()
            if verbose:
                print("reading line {} >".format(num + 1), add_quotes(line))

            if not line:
                if verbose:
                    print(black("Skipping blank line\n", bold=True))
                continue

            if line.startswith("#"):
                if verbose:
                    print(black("Skipping comment line\n", bold=True))
                continue

            # if you find the comment symbol, ignore everything after it
            comment_pos = line.find("#")
            if comment_pos != -1:
                if verbose:
                    print(black("\tRemoving comment > ", bold=True), end='')
                    print(add_quotes(line[comment_pos:]))
                    print(black("\tParsing remaining line > ", bold=True),
                          end='')
                    print(add_quotes(line[:comment_pos]))
                line = line[:comment_pos]

            if count == 0 and steps == 0:
                found_steps = False
                try:
                    nums_in_line = re.findall(r'\d+', line)
                    if len(nums_in_line) == 1:
                        steps = int(nums_in_line[0])
                        found_steps = True
                except (ValueError, IndexError):
                    pass
                if verbose:
                    if found_steps:
                        print(green("\tFound step count:", bold=True), steps)
                    else:
                        print(
                            yellow(
                                "\tFailed to find step count before first interval.",
                                bold=True))
                        print(
                            black("\t\tSetting step count to default:",
                                  bold=True), DEFAULT_STEPS_IN_TIMELINE)
                if found_steps:
                    continue

            intvs = line.split(",")
            timeline.append([])

            for intv in intvs:

                intv = intv.strip()

                # see if it's in the form [#] [#] [#]
                if not parse_check_format(intv, verbose):
                    errors = True
                    continue

                params = intv.split()

                # check that each number is legit
                if not parse_check_numbers(params, verbose):
                    errors = True
                    continue

                # use those valid numbers to make an interval
                new_interval = Interval(int(params[0]), int(params[1]),
                                        int(params[2]))

                if verbose:
                    print(green("\t\tinterval >"),
                          interval_to_string(new_interval))

                total = total + 1
                timeline[count].append(new_interval)

            # if it's run through the line and not added any intervals...
            if not timeline[count]:
                if verbose:
                    print(yellow("no intervals found. Skipping line."))
                del timeline[-1]
            else:
                if verbose:
                    print(green("intervals found:"), len(timeline[count]))
                count = count + 1

            if verbose:
                print()  # newline

    if verbose:
        print("reached end of file.")
        print("found {} intervals across {} channels.".format(
            total, len(timeline)))

    if steps == 0:
        steps = DEFAULT_STEPS_IN_TIMELINE

    return timeline, steps, errors
コード例 #39
0
    ExpectedFile(".top", "Top copper layer", "RS-274X", checkCopperLayer),
    ExpectedFile(".bot", "Bottom copper layer", "RS-274X", checkCopperLayer),
    ExpectedFile(".smt", "Solder mask top", "RS-274X", checkGerberFile),
    ExpectedFile(".smb", "Solder mask bottom", "RS-274X", checkGerberFile),
    ExpectedFile(".plt", "Silk screen top", "RS-274X", checkGerberFile),
    ExpectedFile(".mil", "Board outline", "RS-274X", checkBoardOutline),
    #Drilling
    ExpectedFile(".pth", "Plated through holes", "Excellon", checkExcellonMetric),
    ExpectedFile(".npth", "Non-plated through holes", "Excellon", checkExcellonMetric),
]

if __name__ == "__main__":
    #Parse commandline arguments
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument("directory", help="The directory to scan for project Gerber file")
    parser.add_argument("--gerbv", action="store_true", help="Run gerbv on the files")
    args = parser.parse_args()
    #Perform check
    files = os.listdir(args.directory)
    projectName = extractProjectPrefix(files)
    print(black("Project name: %s" % projectName))
    checkedFiles = [checkFile(args.directory, f, projectName) for f in expectedFiles]
    unknownFiles = set(files) - set(checkedFiles)
    if unknownFiles:
        print(red("Found unknown files: %s" % ",".join(unknownFiles)))
    #Open viewer if enabled
    if args.gerbv:
        filePaths = [os.path.join(args.directory, f) for  f in files]
        subprocess.call(["gerbv"] + filePaths)
コード例 #40
0
        black("Fetching {0} dubbed video URLs for lang {1}".format(
            len(dubbedVideoIDs), lang),
              bold=True))
    fn = functools.partial(getTranslatedVideoId, lang=lang)
    videoURLs = pool.map(fn, dubbedVideoIDs)

    # Remap
    return {
        videoId: url
        for videoId, url in zip(dubbedVideoIDs, videoURLs) if url is not None
    }


if __name__ == "__main__":
    #Download exercise list
    print(black("Downloading master exercise list...", bold=True))
    exercises = getExercises()
    #Download videos
    print(
        black("Downloading exercise video list for {0} exercises...".format(
            len(exercises)),
              bold=True))
    pool = Pool(32)
    exVideos = pool.map(getExerciseVideos, [e["name"] for e in exercises])
    with open("fo.json", "w") as outf:
        json.dump(exVideos, outf)
    # Perform mapping
    print(black("Mapping videos...", bold=True))
    result = []
    allVideoIDs = set()  # Need that for subtitle mapping
    for exercise, videos in zip(exercises, exVideos):
コード例 #41
0
def checkBoardOutline(self, filepath):
    filename = os.path.basename(filepath)
    #Basic gerber checks
    checkGerberFile(self, filepath)
    #Compute board outline
    millLines = readFileLines(filepath)
    # Find factors to get absolute coordinates:
    x_factor, y_factor = findCoordinateFormat(millLines)
    # Initialize X & Y
    x, y = 0, 0
    #We can only interpret the file if coordinates are absolute
    if not "G90*" in millLines:
        print(yellow("Mill coordinates in %s don't seem to be absolute (G90 missing!)" % filename))
        return
    #Determine coordinate units
    unit = parseGerberUnit(millLines)
    if unit is None: #Neither inch nor mm found
        print(yellow("Could not find coordinate units (mm/in) in %s" % filename))
        return
    #Parse the aperture list
    apertures = parseGerberApertures(millLines)
    selectApertureRegex = re.compile(r"(D\d+)\*")
    move2DRegex = re.compile(r"X(-?\d+)Y(-?\d+)D(\d+)\*") #Move (D2) or draw (D1)
    move1DRegex = re.compile(r"([XY])(-?\d+)D(\d+)\*") #With only one coordinate
    #Try to interpret gerber file
    minCoords = (sys.maxsize, sys.maxsize)
    maxCoords = (0, 0)
    lastCoords = (0, 0)
    currentAperture = None
    apertureUseCount = Counter()
    for line in millLines:
        if selectApertureRegex.match(line):
            apertureCode = selectApertureRegex.match(line).group(1)
            currentAperture = findAperture(apertures, apertureCode)
        elif move2DRegex.match(line):
            match = move2DRegex.match(line)
            x = int(match.group(1)) / x_factor
            y = int(match.group(2)) / y_factor
            apertureUseCount[currentAperture] += 1
        elif move1DRegex.match(line):
            match = move1DRegex.match(line)
            apertureUseCount[currentAperture] += 1
            if match.group(1) == "X":
                x = int(match.group(2)) / x_factor
                y = lastCoords[1]
            elif match.group(1) == "Y":
                x = lastCoords[0]
                y = int(match.group(2)) / y_factor
            else: raise Exception("Internal error: Invalid coordinate type in 1D move: %s" % match.group(1))
        else: continue
        #Compute min/max coordinates
        lastCoords = (x, y)
        minCoords = (min(minCoords[0], lastCoords[0]), min(minCoords[1], lastCoords[1]))
        maxCoords = (max(maxCoords[0], lastCoords[0]), max(maxCoords[1], lastCoords[1]))
    #Compute board size (minimum enclosing rectangle)
    boardSize = (maxCoords[0] - minCoords[0], maxCoords[1] - minCoords[1])
    # Compute size of most common aperture
    mostCommonAperture = apertureUseCount.most_common(1)[0][0]
    #   info
    print(black("\tGerber offset: ({1:.2f} {0}, {2:.2f} {0})".format(unit, minCoords[0], minCoords[1])))
    print(black("\tBoard size (minimum rectangle): %.1f %s x %.1f %s" % \
            (boardSize[0], unit, boardSize[1], unit)))
コード例 #42
0
def process_xliff_soup(filename,
                       soup,
                       autotranslator,
                       indexer,
                       autotranslate=True,
                       preindex=False,
                       overwrite=False,
                       postproc=identity):
    """
    Remove both untranslated and notes from the given soup.
    For the untranslated elements, in
    """
    overall_count = 0
    untranslated_count = 0
    translated_count = 0
    autotranslated_count = 0
    # Iterate over all translatable strings
    body = soup.xliff.file.body

    # Resulting elements
    results = []

    indexFN = indexer.preindex if preindex else indexer.add

    for trans_unit in body.children:  #body.find_all("trans-unit"):
        # Ignore strings
        if not isinstance(trans_unit, bs4.element.Tag):
            continue

        # Ignore other tags
        if trans_unit.name != "trans-unit":
            print("Encountered wrong tag: {}".format(trans_unit.name))
            continue

        overall_count += 1
        source = trans_unit.source
        target = trans_unit.target
        # Broken XLIFF?
        if target is None:
            print(trans_unit.prettify())
            continue
        note = trans_unit.note
        is_untranslated = ("state" in target.attrs
                           and target["state"] == "needs-translation")
        is_approved = ("approved" in trans_unit.attrs
                       and trans_unit["approved"] == "yes")

        can_overwrite = not is_untranslated and not is_approved
        will_overwrite = overwrite and can_overwrite

        engl = source.text
        translated = target.text
        # Index tags in the indexer (e.g. to extract text tags)
        # This is done even if they are translated
        # NOTE: This does index or preindex (chosen outside of the loop)
        indexFN(engl,
                None if is_untranslated else translated,
                filename=filename,
                approved=is_approved)

        # For indexing run, ignore autotranslator altogether
        if not autotranslate and not will_overwrite:
            trans_unit.decompose()
            continue

        # Remove entire tag if translated (or suggested)
        if not is_untranslated and not will_overwrite:
            trans_unit.decompose()
            translated_count += 1
            continue  # Dont try to autotranslate etc

        untranslated_count += 1
        # Remove HUGE note text to save space
        if note:
            note.decompose()
        # Remove empty text inside the <trans-unit> element to save space
        for c in trans_unit.contents:
            c.extract()

        # Now we can try to autotranslate
        try:
            autotrans = postproc(autotranslator.translate(engl))
        except:
            print(
                black("Autotranslate fail for string '{}'".format(engl),
                      bold=True))
            traceback.print_exception(*sys.exc_info())
            autotrans = None

        if autotrans is None:  # Could not translate
            # Remove from output file to conserve space
            trans_unit.decompose()
        else:  # Could autotranslate
            # Store autotranslation in XML
            target["state"] = "translated"
            target.string = autotrans
            autotranslated_count += 1
            # Add to result list
            results.append(trans_unit.extract())

    # Remove empty text content of the body to conserve spce
    body.contents = results

    # Print stats
    if autotranslate:
        if untranslated_count != 0:  # Don't print "0 of 0 strings"
            print(
                black(
                    "Autotranslated {} of {} untranslated strings ({} total) in {}"
                    .format(autotranslated_count, untranslated_count,
                            overall_count, os.path.basename(filename))))
    else:
        print(
            black("{} {} strings in {}".format(
                "Preindexed" if preindex else "Indexed", overall_count,
                os.path.basename(filename))))

    return autotranslated_count
コード例 #43
0
    print(black("Fetching dubbed video list for lang {0}".format(lang), bold=True))
    dubbedVideoIDs = findDubbedVideos(lang)

    print(black("Fetching {0} dubbed video URLs for lang {1}"
           .format(len(dubbedVideoIDs), lang), bold=True))
    fn = functools.partial(getTranslatedVideoId, lang=lang)
    videoURLs = pool.map(fn, dubbedVideoIDs)
    
    # Remap
    return {videoId: url
             for videoId, url in zip(dubbedVideoIDs, videoURLs)
             if url is not None}

if __name__ == "__main__":
    #Download exercise list
    print(black("Downloading master exercise list...", bold=True))
    exercises = getExercises()
    #Download videos
    print(black("Downloading exercise video list for {0} exercises..."
        .format(len(exercises)), bold=True))
    pool = Pool(32)
    exVideos = pool.map(getExerciseVideos, [e["name"] for e in exercises])
    with open("fo.json","w") as outf:json.dump(exVideos, outf)
    # Perform mapping
    print(black("Mapping videos...", bold=True))
    result = []
    allVideoIDs = set()  # Need that for subtitle mapping
    for exercise, videos in zip(exercises, exVideos):
        current_videos = []
        for video in videos:
            current_videos.append({