Ejemplo n.º 1
0
 def __init__(self, username, password):
     self.username = username
     self.password = password
     lastw = 0
     if utils.fileExists("cookie.tmp"):
         lastw = time.time() - os.stat("cookie.tmp").st_mtime
     if not utils.fileExists("cookie.tmp") or lastw > 864000:
         self._getSessionID()
         self._login()
     else:
         f = open("cookie.tmp", "r")
         self.session = f.read()
         f.close()
Ejemplo n.º 2
0
    def buildImages(self):
        self.images = {}
        se = self.projectSettings
        readInfo = [
            (IMAGE_CROP_FIELD, cv.IMREAD_COLOR),
            (IMAGE_NORM_FIELD, cv.IMREAD_COLOR),
            (IMAGE_VEG_DENSITY, cv.IMREAD_COLOR),
            (IMAGE_VEG_MASK, cv.IMREAD_GRAYSCALE),
            (IMAGE_WEED_DENSITY, cv.IMREAD_COLOR),
            (IMAGE_WEED_MASK, cv.IMREAD_GRAYSCALE),
            (IMAGE_ROI_MASK, cv.IMREAD_GRAYSCALE),
        ]
        for name, flags in readInfo:
            fileName = '_'.join(name.lower().split()) + '.png'
            filePath = os.path.join(se.projectPath, fileName)
            self.images[name] = Layer(name=name,
                                      filePath=filePath,
                                      flags=flags)

        if not utils.fileExists(self.images[IMAGE_CROP_FIELD].filePath):
            image = cv.imread(se.cropFieldImagePath, cv.IMREAD_UNCHANGED)
            if image.ndim == 3 and image.shape[2] == 4:
                alpha = image[:, :, 3]
                image = image[:, :, 0:3]
                image[alpha < 200] = 0
            self.images[IMAGE_CROP_FIELD].image = image
            self.images[IMAGE_CROP_FIELD].save()
Ejemplo n.º 3
0
    def read(self):
        if utils.fileExists(self.filePath):
            self.image: np.ndarray = cv.imread(self.filePath, flags=self.flags)
            if self.image is None:
                return

            try:
                with open(utils.swapExt(self.filePath, '.im')) as fp:
                    data: dict = json.load(fp)
                    self.name = data.get('name', self.name)
                    self.position = data.get('position', self.position)
                    self.scale = data.get('scale', self.scale)
                    self.colormap = data.get('colormap', self.colormap)
                    self.maprange = data.get('maprange', self.maprange)
                    self.transform = data.get('transform', self.transform)
                    self.flags = data.get('flags', self.flags)
                    shapesData = data.get('shapes', None)
                    if shapesData is not None:
                        for name, shapeSet in shapesData.items():
                            self.shapes[name] = []
                            for shapeData in shapeSet:
                                shape = Shape()
                                shape.data = shapeData
                                self.shapes[name].append(shape)

            except OSError:
                pass
            except Exception as err:
                logger.error(err)
Ejemplo n.º 4
0
    def updateRecentProjectsActions(self):

        menu = self.ui.openRecentAct.menu()
        if menu.isEmpty():
            for i in range(MAX_RECENT_FILES):
                action = QtWidgets.QAction(self)
                action.visible = False
                action.triggered.connect(self.openRecentProject)
                menu.addAction(action)

        recentPaths = self.appSettings.recentFiles
        recentPaths[:] = [
            file for file in recentPaths if utils.fileExists(file)
        ]
        self.appSettings.recentFiles = recentPaths

        if len(recentPaths) == 0:
            self.ui.openRecentAct.setDisabled(True)
            return
        self.ui.openRecentAct.setDisabled(False)

        numRecentFiles = min(len(recentPaths), MAX_RECENT_FILES)
        actions = menu.actions()
        for i in range(numRecentFiles):
            fileName = os.path.basename(recentPaths[i])
            text = f'&{i + 1} {fileName}'
            actions[i].setText(text)
            actions[i].setData(recentPaths[i])
            actions[i].setVisible(True)

        for j in range(numRecentFiles, MAX_RECENT_FILES):
            actions[j].setVisible(False)
Ejemplo n.º 5
0
def loadMFCCSValues(file_names):
    if not utils.fileExists(config.OUTPUT_CSV_FILE):  # if there is no CSV file containing all info
        mfccs = sound_tools.getMFCCS(file_names)
        writing_info = utils.getCSVWriteInfo(file_names, mfccs)
        utils.writeToCSV(writing_info)
        return utils.fixArray(mfccs)
    else:  # if it exists, then just load the values from it instead of computing them again
        return utils.loadCSVInfo()
Ejemplo n.º 6
0
    def loadAppSettings(self):

        try:
            self.appSettings = AppSettings.load()
        except Exception as err:
            logger.error(err)

        self.ui.pos = self.appSettings.mainWindowPos
        self.ui.size = self.appSettings.mainWindowSize

        lastProjectPath = self.appSettings.lastProjectPath
        if utils.fileExists(lastProjectPath):
            self.loadProjectFile(lastProjectPath)
        self.updateRecentProjectsActions()
Ejemplo n.º 7
0
 def feedIndexer( self, folder, name, data, podcasts ):
     feedList = []
     for item, values in data.iteritems(): 
         podcast = os.path.join( folder, values[ "fileName" ] )
         if utils.fileExists( podcast ):
             status = "got"
         elif podcast in podcasts:
             status = "had"
         else:
             status = "need"
         feedList.append( {
             "name" : name,
             "title" : values[ "title" ],
             "podcast" : podcast,
             "status"  : status,
             "fileName" : values[ "fileName" ],
         } )
     return feedList
Ejemplo n.º 8
0
    wb = Workbook()
    sheet = wb.create_sheet("Worksheet")
    for i in range(len(rows)):
        row = rows[i]
        for j in range(len(row)):
            cell = row[j]

            spec = getAlpha[j] + str(i + 1)
            sheet[spec] = cell

    wb.remove(wb["Sheet"])
    wb.save(filename)


# Check for db
if (not (utils.fileExists(DB_LOCATION))):
    raise Exception("Please add a database at '" + DB_LOCATION + "'")

# Create input folder and subfolders
utils.check_and_create_path("input/articles")
utils.check_and_create_path("input/illustrations")

utils.safeWrite(
    "input/articles/README.md",
    "Place articles here. Articles should be included in a text file and the filename should be [lemma].txt, where [lemma] is the lemma the article is about. For example, the article for ἄβουλος should be in ἄβουλος.txt"
)
utils.safeWrite(
    "input/illustrations/README.md",
    "Place illustrations here. Illustrations should be a .jpg, .gif, or .png with the name of the lemma they are an illustraiton for. For example, the image for ἄβουλος should be in ἄβουλος.png (or ἄβουλος.gif or ἄβουλος.jpg)"
)
Ejemplo n.º 9
0
 def isEmpty(self):
     return not utils.fileExists(self.filePath)
def meta_assist(region, small_buf, large_buf):

    # NEW as of 12/01/21: avoid re-computing data that already exists.
    #                     This means that whenever OSM_jcts.py is executed
    #                     (and also when manual editing is performed and
    #                     'manualMergeTool' > save_result is called),
    #                     the resultant data is not only written to .csv but
    #                     also saved as a pickle (= python serialization format
    #                     that is easy to read in, i.e. as opposed to a .csv
    #                     file we don't have to parse numerical data, lists etc
    #                     from string)
    #                     => rather than calling OSM_jcts.py for small_buf and
    #                        large_buf per default, check if we already have the
    #                        data and only compute if we don't.

    # Get data frames for small_buf (more conservative buffer parameter) and large_buf (more liberal buffer parameter)
    # => from pickle (PyPipeline_/junctions/pickled_data) or computed

    # Do we already have a data set for the SMALL buffer size specified?
    # => if so, use it. Else, compute it.

    small_buf_file = f"{region}_junctions_buffer={small_buf}"

    if (utils.fileExists(small_buf_file)):

        small_buf_path = utils.getSubDirPath(small_buf_file, "pickled_data",
                                             "junctions")

        small_buf = pd.read_pickle(small_buf_path)

    else:
        small_buf = OSM_jcts.main(region, small_buf)

    # Do we already have a data set for the LARGE buffer size specified?
    # => if so, use it. Else, compute it.

    large_buf_file = f"{region}_junctions_buffer={large_buf}"

    if (utils.fileExists(large_buf_file)):

        large_buf_path = utils.getSubDirPath(large_buf_file, "pickled_data",
                                             "junctions")

        large_buf = pd.read_pickle(large_buf_path)

    else:
        large_buf = OSM_jcts.main(region, large_buf)

    # Determine where the two data frames (and thus the clustering solutions for smaller and larger buffer) differ

    small_buf_inconsist, small_buf_consist, large_buf_inconsist = determine_inconsistencies(
        small_buf, large_buf)

    mapping.runAllMapTasks(region, small_buf_inconsist, large_buf_inconsist)

    # PICKLE (SERIALIZE) THREE DATA SETS FOR USE BY MANUALMERGETOOL:
    # (1) 'small_buf_inconsist': subset of the small buffer df where clustering solutions differ from the
    #     larger buffer solution.
    # (2) 'large_buf_inconsist': subset of the large buffer df where clustering solutions differ from the
    #     smaller buffer solution.
    # (3) 'consistent_clusters': subset of the small buffer df where clustering solutions DO NOT differ from
    #     the larger buffer solution; i.e. if this subset was taken from the large buffer df it would be exactly
    #     the same.

    # INTENDED PROCESSING OF THESE DATA SETS IN MANUALMERGETOOL:
    # * The more conservative solutions contained in 'small_buf_inconsist' can be manually edited, i.e. replaced by the
    #   more liberal solutions contained in 'large_buf_inconsist'.
    # * That means, when a user compares the rivaling conservative vs. liberal solutions for inconsistent clusters,
    #   she might decide to pick the liberal solution over the conservative one.
    # * Hence, the respective rows belonging to the conservative solution are DELETED from the 'small_buf_inconsist'
    #   df and the respective row belonging to the liberal solution is taken from the 'large_buf_inconsist' data set
    #   and MOVED to 'small_buf_consist', our 'base' df which will be returned after all of the manual editing is
    #   finished. This means that the conflict has been resolved.
    # * When all editing is done, what remains of 'small_buf_inconsist' (i.e., the conservative solutions that
    #   were chosen over their liberal counterparts) is concatenated with 'consistent_clusters', which already
    #   contains all the more liberal solutions that were chosen over the conservative ones.

    # Write small_buf_inconsist pickle

    small_buf_inconsist_path = utils.getSubDirPath(
        f"jcts_small_buf_inconsist_{region}", "pickled_data", "junctions")

    small_buf_inconsist.to_pickle(small_buf_inconsist_path)

    # Write large_buf_inconsist pickle

    large_buf_inconsist_path = utils.getSubDirPath(
        f"jcts_large_buf_inconsist_{region}", "pickled_data", "junctions")

    large_buf_inconsist.to_pickle(large_buf_inconsist_path)

    # Write consistent clusters pickle

    consistent_clusters_path = utils.getSubDirPath(
        f"jcts_consistent_clusters_{region}", "pickled_data", "junctions")

    small_buf_consist.to_pickle(consistent_clusters_path)
Ejemplo n.º 11
0
def keyAuthorComparisonWithImportance(authors, books, baseSaveDir, splitParam,
                                      topWords):
    makeWordImportanceGraphs = False
    keyAuthData = getKeyAuthorData(authors, books)
    saveDir = baseSaveDir + "wordImportance/"
    allDiffLineData = {}
    allCumulDiffLineData = {}
    allRCumulDiffLineData = {}
    allPercentageLineData = {}

    # load diffs for plotting internal similarities
    allDiffsFilename = baseSaveDir + "dists/diffLists.json"
    allDiffs = utils.getContent(allDiffsFilename, True)

    # For each set of key authors, make necessary visaulizations
    for dat in keyAuthData:
        data, _, dataLabels, chartFileName = dat

        print("    %s..." % chartFileName)
        numWords = len(topWords)
        numTexts = len(dataLabels)
        tickLabels = topWords
        distsFilename = baseSaveDir + "dists/" + chartFileName + ".json"
        dists = utils.getContent(distsFilename, True)
        # dists = [
        #     {"name": "D1", "vals": (np.random.random((numWords))*1.5 - 0.5)},
        #     {"name": "D2", "vals": (np.random.random((numWords))*1.5 - 0.5)}
        # ]
        for d in dists:
            d["vals"] = np.array(d["vals"])

        if (makeWordImportanceGraphs):
            graphUtils.wordImportanceComparison(data, dataLabels, tickLabels,
                                                dists, saveDir + "unsorted/",
                                                chartFileName, True)

        # display versions sorted by each metric
        for d in dists:
            sortedSaveDir = saveDir + d["name"] + "-sorted/"
            fname = chartFileName
            sortedInds = np.array(
                list(
                    map(
                        lambda x: x[0],
                        sorted(enumerate(d["vals"]),
                               key=lambda x: x[1][0],
                               reverse=True))))

            data1 = copy.deepcopy(data)
            tickLabels1 = copy.deepcopy(tickLabels)
            wordsUsed = len(topWords)
            # If the similarity metric includes remainder, we have to add it
            if (len(dists[0]["vals"]) == len(data[0]) + 1):
                newData = []
                for row in data1:
                    r = np.append(row, 1 - np.sum(row))
                    newData.append(r)
                data1 = newData

                tickLabels1.append("Remainder")
                wordsUsed += 1

            data2 = list(map(lambda x: np.array(x)[sortedInds], data1))
            tickLabels2 = np.array(tickLabels1)[sortedInds]
            dists2 = copy.deepcopy(dists)
            percentiles = []
            for d2 in dists2:
                d2["vals"] = np.copy(d2["vals"])[sortedInds]

            if (makeWordImportanceGraphs):
                graphUtils.wordImportanceComparison(data2, dataLabels,
                                                    tickLabels2, dists2,
                                                    sortedSaveDir, fname, True)

            # save all words
            if d["name"] == "Jensen-shannon":
                fname = saveDir + "keyWords/" + chartFileName + ".json"
                SimDiff = {}
                for i, val in enumerate(d["vals"][sortedInds]):
                    if (True):
                        SimDiff[tickLabels2[i]] = [i, val[1]]
                utils.safeWrite(fname, SimDiff, True)

            # Diff data
            trueDiffs = np.array(
                list(map(lambda x: x[0], d["vals"][sortedInds])))
            y = (chartFileName, trueDiffs)
            y_cumul = (chartFileName, np.cumsum(trueDiffs))
            linesToGraphDiff = [y]
            linesToGraphDiffCumul = [y_cumul]

            # store info for the chart with all authors
            if d["name"] in allDiffLineData:
                allDiffLineData[d["name"]].extend([y])
            else:
                allDiffLineData[d["name"]] = [y]
            if d["name"] in allCumulDiffLineData:
                allCumulDiffLineData[d["name"]].extend([y_cumul])
            else:
                allCumulDiffLineData[d["name"]] = [y_cumul]

            # dif percentile data
            percentiles = list(map(lambda x: x[1], d["vals"][sortedInds]))
            y = (chartFileName, percentiles)
            linesToGraphPct = [y]

            # store info for the chart with all authors
            if d["name"] in allPercentageLineData:
                allPercentageLineData[d["name"]].append(y)
            else:
                allPercentageLineData[d["name"]] = [y]

            if splitParam == -1:
                # get percentiles for internal consistency of second author
                author1 = dataLabels[0]
                author2 = dataLabels[1]

                authorInternalConsistencies = [
                    # ["split5", author1, "-split5"],
                    # ["split-2", author1, "-splitHalf"],

                    # ["split5", author2, "-split5"],
                    # ["split-2", author2, "-splitHalf"]
                ]

                # Gen information comparing consistencies within given authors.
                for aic in authorInternalConsistencies:
                    a2DiffsFilename = baseSaveDir.replace(
                        "no_split",
                        aic[0]) + "dists/%s_%s_2.json" % (aic[1], aic[1])
                    if (utils.fileExists(a2DiffsFilename)):
                        a2Diffs = utils.getContent(a2DiffsFilename, True)
                        diffNums = None
                        for ad in allDiffs:
                            if ad["name"] == d["name"]:
                                diffNums = ad["allDiffs"]

                        a2RawDiffs = None
                        for ad in a2Diffs:
                            if ad["name"] == d["name"]:
                                a2RawDiffs = ad["vals"]

                        if (diffNums != None and a2RawDiffs != None):
                            # Add difference data
                            aicName = aic[1] + aic[2]
                            a2SortedInds = np.array(
                                list(
                                    map(
                                        lambda x: int(x[0]),
                                        sorted(enumerate(a2RawDiffs),
                                               key=lambda x: x[1][0],
                                               reverse=True))))
                            trueDiffs = np.array(
                                list(
                                    map(lambda x: x[0],
                                        np.array(a2RawDiffs)[a2SortedInds])))
                            y_diff = (aicName, trueDiffs)
                            y_diff_cumul = (aicName, np.cumsum(trueDiffs))
                            linesToGraphDiff.append(y_diff)
                            linesToGraphDiffCumul.append(y_diff_cumul)

                            # Add Percentile data
                            a2Percentiles = []
                            for rd in a2RawDiffs:
                                index = bisect.bisect_left(diffNums, rd[0])
                                a2Percentiles.append(
                                    (100.0 * index) / len(diffNums))

                            a2Percentiles = sorted(a2Percentiles, reverse=True)
                            y2 = (aicName, a2Percentiles)
                            linesToGraphPct.append(y2)
                    else:
                        print("File does not exist: \"%s\"" % a2DiffsFilename)

            # Create charts showing differences for various authors
            graphUtils.lineChart(range(wordsUsed),
                                 linesToGraphDiff,
                                 True,
                                 sortedSaveDir,
                                 chartFileName + "_diff-chart",
                                 yLim=None)  #[-0.002, 0]
            graphUtils.lineChart(range(wordsUsed),
                                 linesToGraphDiffCumul,
                                 True,
                                 sortedSaveDir,
                                 chartFileName + "_diff-cumul-chart",
                                 yLim=None,
                                 yAdjust=1)  #[-0.002, 0]
            #graphUtils.lineChart(range(wordsUsed), linesToGraphPct, True, sortedSaveDir, chartFileName+"_pct-chart")

            linesToGraphDiffRCumul = []
            for name, c in linesToGraphDiffCumul:
                name = name.replace("-split5", " Local Split")
                name = name.replace("-splitHalf", " Global Split")
                linesToGraphDiffRCumul.append((name, c[-1] - np.array(c)))

            if d["name"] in allRCumulDiffLineData:
                allRCumulDiffLineData[d["name"]].extend(
                    [linesToGraphDiffRCumul])
            else:
                allRCumulDiffLineData[d["name"]] = [linesToGraphDiffRCumul]
            graphUtils.lineChart(range(wordsUsed),
                                 linesToGraphDiffRCumul,
                                 True,
                                 sortedSaveDir,
                                 chartFileName + "_diff-r-cumul-chart",
                                 yLim=None,
                                 yAdjust=1)  #[-0.002, 0]

    for d in dists:
        # 4-Up Chart for these authors
        sortedSaveDir = saveDir + d["name"] + "-sorted/"
        graphUtils.lineChart4Up(range(wordsUsed),
                                allRCumulDiffLineData[d["name"]],
                                True,
                                sortedSaveDir,
                                "4up-r-cumul",
                                yLim=None,
                                yAdjust=1)

    # Create graph charts for all data in a cloud
    graphTypes = [
        ("all-diffs", allDiffLineData, None, 0),
        ("all-diffs-cumul", allCumulDiffLineData, None, 1),
        #("all-pcts", allPercentageLineData, [0, 100], 0)
    ]
    alls = {}
    for graphType, lineList, yLim, adjust in graphTypes:
        medFilename = baseSaveDir + "dists/median-%s.json" % graphType
        med = utils.getContent(medFilename, True)

        alls[graphType] = {}
        for d in med:
            lineList[d["name"]].append(["Median", d["line"]])
            alls[graphType][d["name"]] = d["all"]

        for name in allPercentageLineData:
            sortedSaveDir = baseSaveDir + "wordImportance/" + name + "-sorted/"
            for log in [False]:  #, True]:
                print("  %s..." % graphType)
                graphUtils.lineChart(range(wordsUsed),
                                     lineList[name],
                                     True,
                                     sortedSaveDir,
                                     graphType,
                                     yLim=yLim,
                                     log=log,
                                     yAdjust=adjust)
                print("  %s cloud..." % graphType)
                graphUtils.lineChart(range(wordsUsed),
                                     lineList[name],
                                     True,
                                     sortedSaveDir,
                                     graphType + "-cloud",
                                     yLim=yLim,
                                     allLines=alls[graphType][name],
                                     log=log,
                                     yAdjust=adjust)

    # Create chart showing ignored top words
    n = "Jensen-shannon"
    sortedSaveDir = baseSaveDir + "wordImportance/" + n + "-sorted/"

    # Cumulative
    data = allCumulDiffLineData[n]

    # Add lines
    res = []
    targetSim = -1
    for item in alls["all-diffs-cumul"][n]:
        name, c = item
        # "Aristotle_Pindar" in name or

        #"AeliusAristides_Demosthenes", "DioChrysostom_Plato"
        if ("ApolloniusRhodius_QuintusSmyrnaeus" in name
                or "DioChrysostom_Xenophon" == name):
            res.append((name, "-", 1 + c[-1] - np.array(c)))

        # Lowest of our top authors
        if ("DioChrysostom_Xenophon" == name):
            targetSim = c[-1]

    # add median
    # for item in allCumulDiffLineData[n]:
    #     name, c = item
    #     if ("Median" in name):
    #         res.append((name, "-", 1 + c[-1] - np.array(c)))

    # Add line cloud
    resAll = []
    for item in alls["all-diffs-cumul"][n]:
        name, c = item
        if not ("Hymns_Dionysus" in name or "Euclid" in name):
            n1, n2 = name.replace("Hymns_", "Hymns").split("_")
            n1 = n1.replace("Hymns", "Hymns_")
            n2 = n2.replace("Hymns", "Hymns_")
            centuryDiff = centDiff(genre.toCent(n1), genre.toCent(n2))
            #print("%s, %s: %d" % (n1, n2, centuryDiff))
            if (centuryDiff >= 4):
                # color top sims differently
                color = "k-"

                resAll.append((name, color, 1 + c[-1] - np.array(c)))

    # for name, c in data:
    #     y = c[-1] - np.array(c)
    #     res.append((name, y))

    #resAll = map(lambda n, c: (n, c[-1] - np.array(c)))
    graphUtils.compareWordUsageChart(res,
                                     True,
                                     sortedSaveDir,
                                     "ignoreBestWords",
                                     yLim=None,
                                     allLines=resAll)
Ejemplo n.º 12
0
import xml.dom.minidom, time, re, select, nntp, socket
import os, sys, string, ConfigParser, codecs, utils, time
import downloader, verifier, extractor, bookmarks, nntplib

# check nzb files
nfiles = []
if len(sys.argv) > 1:
    for arg in sys.argv[1:]:
        if utils.fileExists(arg):
            if arg.split(".")[-1].lower() == "nzb":
                nfiles.append({"path": arg, "isBookmark": False})
                if arg[0] == "/":
                    # unix style
                    print "added [%s] to the queue" % string.split(arg, "/")[-1]
                else:
                    # win32 style
                    print "added [%s] to the queue" % string.split(arg, "\\")[-1]
            else:
                print "not an nzb file: %s" % arg

# change the cwd
# NOTE: with py2exe use sys.executable instead of __file__
approot = os.path.dirname(__file__)
if approot != '':
    os.chdir(approot)

# load settings
cfg = ConfigParser.ConfigParser()
cfg.read("settings.conf")

# load gui