def __init__(self, username, password):
self.username = username
self.password = password
lastw = 0
if utils.fileExists("cookie.tmp"):
lastw = time.time() - os.stat("cookie.tmp").st_mtime
if not utils.fileExists("cookie.tmp") or lastw > 864000:
self._getSessionID()
self._login()
else:
f = open("cookie.tmp", "r")
self.session = f.read()
f.close()
def buildImages(self):
self.images = {}
se = self.projectSettings
readInfo = [
(IMAGE_CROP_FIELD, cv.IMREAD_COLOR),
(IMAGE_NORM_FIELD, cv.IMREAD_COLOR),
(IMAGE_VEG_DENSITY, cv.IMREAD_COLOR),
(IMAGE_VEG_MASK, cv.IMREAD_GRAYSCALE),
(IMAGE_WEED_DENSITY, cv.IMREAD_COLOR),
(IMAGE_WEED_MASK, cv.IMREAD_GRAYSCALE),
(IMAGE_ROI_MASK, cv.IMREAD_GRAYSCALE),
]
for name, flags in readInfo:
fileName = '_'.join(name.lower().split()) + '.png'
filePath = os.path.join(se.projectPath, fileName)
self.images[name] = Layer(name=name,
filePath=filePath,
flags=flags)
if not utils.fileExists(self.images[IMAGE_CROP_FIELD].filePath):
image = cv.imread(se.cropFieldImagePath, cv.IMREAD_UNCHANGED)
if image.ndim == 3 and image.shape[2] == 4:
alpha = image[:, :, 3]
image = image[:, :, 0:3]
image[alpha < 200] = 0
self.images[IMAGE_CROP_FIELD].image = image
self.images[IMAGE_CROP_FIELD].save()
def read(self):
if utils.fileExists(self.filePath):
self.image: np.ndarray = cv.imread(self.filePath, flags=self.flags)
if self.image is None:
return
try:
with open(utils.swapExt(self.filePath, '.im')) as fp:
data: dict = json.load(fp)
self.name = data.get('name', self.name)
self.position = data.get('position', self.position)
self.scale = data.get('scale', self.scale)
self.colormap = data.get('colormap', self.colormap)
self.maprange = data.get('maprange', self.maprange)
self.transform = data.get('transform', self.transform)
self.flags = data.get('flags', self.flags)
shapesData = data.get('shapes', None)
if shapesData is not None:
for name, shapeSet in shapesData.items():
self.shapes[name] = []
for shapeData in shapeSet:
shape = Shape()
shape.data = shapeData
self.shapes[name].append(shape)
except OSError:
pass
except Exception as err:
logger.error(err)
def updateRecentProjectsActions(self):
menu = self.ui.openRecentAct.menu()
if menu.isEmpty():
for i in range(MAX_RECENT_FILES):
action = QtWidgets.QAction(self)
action.visible = False
action.triggered.connect(self.openRecentProject)
menu.addAction(action)
recentPaths = self.appSettings.recentFiles
recentPaths[:] = [
file for file in recentPaths if utils.fileExists(file)
]
self.appSettings.recentFiles = recentPaths
if len(recentPaths) == 0:
self.ui.openRecentAct.setDisabled(True)
return
self.ui.openRecentAct.setDisabled(False)
numRecentFiles = min(len(recentPaths), MAX_RECENT_FILES)
actions = menu.actions()
for i in range(numRecentFiles):
fileName = os.path.basename(recentPaths[i])
text = f'&{i + 1} {fileName}'
actions[i].setText(text)
actions[i].setData(recentPaths[i])
actions[i].setVisible(True)
for j in range(numRecentFiles, MAX_RECENT_FILES):
actions[j].setVisible(False)
def loadMFCCSValues(file_names):
if not utils.fileExists(config.OUTPUT_CSV_FILE): # if there is no CSV file containing all info
mfccs = sound_tools.getMFCCS(file_names)
writing_info = utils.getCSVWriteInfo(file_names, mfccs)
utils.writeToCSV(writing_info)
return utils.fixArray(mfccs)
else: # if it exists, then just load the values from it instead of computing them again
return utils.loadCSVInfo()
def loadAppSettings(self):
try:
self.appSettings = AppSettings.load()
except Exception as err:
logger.error(err)
self.ui.pos = self.appSettings.mainWindowPos
self.ui.size = self.appSettings.mainWindowSize
lastProjectPath = self.appSettings.lastProjectPath
if utils.fileExists(lastProjectPath):
self.loadProjectFile(lastProjectPath)
self.updateRecentProjectsActions()
def feedIndexer( self, folder, name, data, podcasts ):
feedList = []
for item, values in data.iteritems():
podcast = os.path.join( folder, values[ "fileName" ] )
if utils.fileExists( podcast ):
status = "got"
elif podcast in podcasts:
status = "had"
else:
status = "need"
feedList.append( {
"name" : name,
"title" : values[ "title" ],
"podcast" : podcast,
"status" : status,
"fileName" : values[ "fileName" ],
} )
return feedList
wb = Workbook()
sheet = wb.create_sheet("Worksheet")
for i in range(len(rows)):
row = rows[i]
for j in range(len(row)):
cell = row[j]
spec = getAlpha[j] + str(i + 1)
sheet[spec] = cell
wb.remove(wb["Sheet"])
wb.save(filename)
# Check for db
if (not (utils.fileExists(DB_LOCATION))):
raise Exception("Please add a database at '" + DB_LOCATION + "'")
# Create input folder and subfolders
utils.check_and_create_path("input/articles")
utils.check_and_create_path("input/illustrations")
utils.safeWrite(
"input/articles/README.md",
"Place articles here. Articles should be included in a text file and the filename should be [lemma].txt, where [lemma] is the lemma the article is about. For example, the article for ἄβουλος should be in ἄβουλος.txt"
)
utils.safeWrite(
"input/illustrations/README.md",
"Place illustrations here. Illustrations should be a .jpg, .gif, or .png with the name of the lemma they are an illustraiton for. For example, the image for ἄβουλος should be in ἄβουλος.png (or ἄβουλος.gif or ἄβουλος.jpg)"
)
def isEmpty(self):
return not utils.fileExists(self.filePath)
def meta_assist(region, small_buf, large_buf):
# NEW as of 12/01/21: avoid re-computing data that already exists.
# This means that whenever OSM_jcts.py is executed
# (and also when manual editing is performed and
# 'manualMergeTool' > save_result is called),
# the resultant data is not only written to .csv but
# also saved as a pickle (= python serialization format
# that is easy to read in, i.e. as opposed to a .csv
# file we don't have to parse numerical data, lists etc
# from string)
# => rather than calling OSM_jcts.py for small_buf and
# large_buf per default, check if we already have the
# data and only compute if we don't.
# Get data frames for small_buf (more conservative buffer parameter) and large_buf (more liberal buffer parameter)
# => from pickle (PyPipeline_/junctions/pickled_data) or computed
# Do we already have a data set for the SMALL buffer size specified?
# => if so, use it. Else, compute it.
small_buf_file = f"{region}_junctions_buffer={small_buf}"
if (utils.fileExists(small_buf_file)):
small_buf_path = utils.getSubDirPath(small_buf_file, "pickled_data",
"junctions")
small_buf = pd.read_pickle(small_buf_path)
else:
small_buf = OSM_jcts.main(region, small_buf)
# Do we already have a data set for the LARGE buffer size specified?
# => if so, use it. Else, compute it.
large_buf_file = f"{region}_junctions_buffer={large_buf}"
if (utils.fileExists(large_buf_file)):
large_buf_path = utils.getSubDirPath(large_buf_file, "pickled_data",
"junctions")
large_buf = pd.read_pickle(large_buf_path)
else:
large_buf = OSM_jcts.main(region, large_buf)
# Determine where the two data frames (and thus the clustering solutions for smaller and larger buffer) differ
small_buf_inconsist, small_buf_consist, large_buf_inconsist = determine_inconsistencies(
small_buf, large_buf)
mapping.runAllMapTasks(region, small_buf_inconsist, large_buf_inconsist)
# PICKLE (SERIALIZE) THREE DATA SETS FOR USE BY MANUALMERGETOOL:
# (1) 'small_buf_inconsist': subset of the small buffer df where clustering solutions differ from the
# larger buffer solution.
# (2) 'large_buf_inconsist': subset of the large buffer df where clustering solutions differ from the
# smaller buffer solution.
# (3) 'consistent_clusters': subset of the small buffer df where clustering solutions DO NOT differ from
# the larger buffer solution; i.e. if this subset was taken from the large buffer df it would be exactly
# the same.
# INTENDED PROCESSING OF THESE DATA SETS IN MANUALMERGETOOL:
# * The more conservative solutions contained in 'small_buf_inconsist' can be manually edited, i.e. replaced by the
# more liberal solutions contained in 'large_buf_inconsist'.
# * That means, when a user compares the rivaling conservative vs. liberal solutions for inconsistent clusters,
# she might decide to pick the liberal solution over the conservative one.
# * Hence, the respective rows belonging to the conservative solution are DELETED from the 'small_buf_inconsist'
# df and the respective row belonging to the liberal solution is taken from the 'large_buf_inconsist' data set
# and MOVED to 'small_buf_consist', our 'base' df which will be returned after all of the manual editing is
# finished. This means that the conflict has been resolved.
# * When all editing is done, what remains of 'small_buf_inconsist' (i.e., the conservative solutions that
# were chosen over their liberal counterparts) is concatenated with 'consistent_clusters', which already
# contains all the more liberal solutions that were chosen over the conservative ones.
# Write small_buf_inconsist pickle
small_buf_inconsist_path = utils.getSubDirPath(
f"jcts_small_buf_inconsist_{region}", "pickled_data", "junctions")
small_buf_inconsist.to_pickle(small_buf_inconsist_path)
# Write large_buf_inconsist pickle
large_buf_inconsist_path = utils.getSubDirPath(
f"jcts_large_buf_inconsist_{region}", "pickled_data", "junctions")
large_buf_inconsist.to_pickle(large_buf_inconsist_path)
# Write consistent clusters pickle
consistent_clusters_path = utils.getSubDirPath(
f"jcts_consistent_clusters_{region}", "pickled_data", "junctions")
small_buf_consist.to_pickle(consistent_clusters_path)
def keyAuthorComparisonWithImportance(authors, books, baseSaveDir, splitParam,
topWords):
makeWordImportanceGraphs = False
keyAuthData = getKeyAuthorData(authors, books)
saveDir = baseSaveDir + "wordImportance/"
allDiffLineData = {}
allCumulDiffLineData = {}
allRCumulDiffLineData = {}
allPercentageLineData = {}
# load diffs for plotting internal similarities
allDiffsFilename = baseSaveDir + "dists/diffLists.json"
allDiffs = utils.getContent(allDiffsFilename, True)
# For each set of key authors, make necessary visaulizations
for dat in keyAuthData:
data, _, dataLabels, chartFileName = dat
print(" %s..." % chartFileName)
numWords = len(topWords)
numTexts = len(dataLabels)
tickLabels = topWords
distsFilename = baseSaveDir + "dists/" + chartFileName + ".json"
dists = utils.getContent(distsFilename, True)
# dists = [
# {"name": "D1", "vals": (np.random.random((numWords))*1.5 - 0.5)},
# {"name": "D2", "vals": (np.random.random((numWords))*1.5 - 0.5)}
# ]
for d in dists:
d["vals"] = np.array(d["vals"])
if (makeWordImportanceGraphs):
graphUtils.wordImportanceComparison(data, dataLabels, tickLabels,
dists, saveDir + "unsorted/",
chartFileName, True)
# display versions sorted by each metric
for d in dists:
sortedSaveDir = saveDir + d["name"] + "-sorted/"
fname = chartFileName
sortedInds = np.array(
list(
map(
lambda x: x[0],
sorted(enumerate(d["vals"]),
key=lambda x: x[1][0],
reverse=True))))
data1 = copy.deepcopy(data)
tickLabels1 = copy.deepcopy(tickLabels)
wordsUsed = len(topWords)
# If the similarity metric includes remainder, we have to add it
if (len(dists[0]["vals"]) == len(data[0]) + 1):
newData = []
for row in data1:
r = np.append(row, 1 - np.sum(row))
newData.append(r)
data1 = newData
tickLabels1.append("Remainder")
wordsUsed += 1
data2 = list(map(lambda x: np.array(x)[sortedInds], data1))
tickLabels2 = np.array(tickLabels1)[sortedInds]
dists2 = copy.deepcopy(dists)
percentiles = []
for d2 in dists2:
d2["vals"] = np.copy(d2["vals"])[sortedInds]
if (makeWordImportanceGraphs):
graphUtils.wordImportanceComparison(data2, dataLabels,
tickLabels2, dists2,
sortedSaveDir, fname, True)
# save all words
if d["name"] == "Jensen-shannon":
fname = saveDir + "keyWords/" + chartFileName + ".json"
SimDiff = {}
for i, val in enumerate(d["vals"][sortedInds]):
if (True):
SimDiff[tickLabels2[i]] = [i, val[1]]
utils.safeWrite(fname, SimDiff, True)
# Diff data
trueDiffs = np.array(
list(map(lambda x: x[0], d["vals"][sortedInds])))
y = (chartFileName, trueDiffs)
y_cumul = (chartFileName, np.cumsum(trueDiffs))
linesToGraphDiff = [y]
linesToGraphDiffCumul = [y_cumul]
# store info for the chart with all authors
if d["name"] in allDiffLineData:
allDiffLineData[d["name"]].extend([y])
else:
allDiffLineData[d["name"]] = [y]
if d["name"] in allCumulDiffLineData:
allCumulDiffLineData[d["name"]].extend([y_cumul])
else:
allCumulDiffLineData[d["name"]] = [y_cumul]
# dif percentile data
percentiles = list(map(lambda x: x[1], d["vals"][sortedInds]))
y = (chartFileName, percentiles)
linesToGraphPct = [y]
# store info for the chart with all authors
if d["name"] in allPercentageLineData:
allPercentageLineData[d["name"]].append(y)
else:
allPercentageLineData[d["name"]] = [y]
if splitParam == -1:
# get percentiles for internal consistency of second author
author1 = dataLabels[0]
author2 = dataLabels[1]
authorInternalConsistencies = [
# ["split5", author1, "-split5"],
# ["split-2", author1, "-splitHalf"],
# ["split5", author2, "-split5"],
# ["split-2", author2, "-splitHalf"]
]
# Gen information comparing consistencies within given authors.
for aic in authorInternalConsistencies:
a2DiffsFilename = baseSaveDir.replace(
"no_split",
aic[0]) + "dists/%s_%s_2.json" % (aic[1], aic[1])
if (utils.fileExists(a2DiffsFilename)):
a2Diffs = utils.getContent(a2DiffsFilename, True)
diffNums = None
for ad in allDiffs:
if ad["name"] == d["name"]:
diffNums = ad["allDiffs"]
a2RawDiffs = None
for ad in a2Diffs:
if ad["name"] == d["name"]:
a2RawDiffs = ad["vals"]
if (diffNums != None and a2RawDiffs != None):
# Add difference data
aicName = aic[1] + aic[2]
a2SortedInds = np.array(
list(
map(
lambda x: int(x[0]),
sorted(enumerate(a2RawDiffs),
key=lambda x: x[1][0],
reverse=True))))
trueDiffs = np.array(
list(
map(lambda x: x[0],
np.array(a2RawDiffs)[a2SortedInds])))
y_diff = (aicName, trueDiffs)
y_diff_cumul = (aicName, np.cumsum(trueDiffs))
linesToGraphDiff.append(y_diff)
linesToGraphDiffCumul.append(y_diff_cumul)
# Add Percentile data
a2Percentiles = []
for rd in a2RawDiffs:
index = bisect.bisect_left(diffNums, rd[0])
a2Percentiles.append(
(100.0 * index) / len(diffNums))
a2Percentiles = sorted(a2Percentiles, reverse=True)
y2 = (aicName, a2Percentiles)
linesToGraphPct.append(y2)
else:
print("File does not exist: \"%s\"" % a2DiffsFilename)
# Create charts showing differences for various authors
graphUtils.lineChart(range(wordsUsed),
linesToGraphDiff,
True,
sortedSaveDir,
chartFileName + "_diff-chart",
yLim=None) #[-0.002, 0]
graphUtils.lineChart(range(wordsUsed),
linesToGraphDiffCumul,
True,
sortedSaveDir,
chartFileName + "_diff-cumul-chart",
yLim=None,
yAdjust=1) #[-0.002, 0]
#graphUtils.lineChart(range(wordsUsed), linesToGraphPct, True, sortedSaveDir, chartFileName+"_pct-chart")
linesToGraphDiffRCumul = []
for name, c in linesToGraphDiffCumul:
name = name.replace("-split5", " Local Split")
name = name.replace("-splitHalf", " Global Split")
linesToGraphDiffRCumul.append((name, c[-1] - np.array(c)))
if d["name"] in allRCumulDiffLineData:
allRCumulDiffLineData[d["name"]].extend(
[linesToGraphDiffRCumul])
else:
allRCumulDiffLineData[d["name"]] = [linesToGraphDiffRCumul]
graphUtils.lineChart(range(wordsUsed),
linesToGraphDiffRCumul,
True,
sortedSaveDir,
chartFileName + "_diff-r-cumul-chart",
yLim=None,
yAdjust=1) #[-0.002, 0]
for d in dists:
# 4-Up Chart for these authors
sortedSaveDir = saveDir + d["name"] + "-sorted/"
graphUtils.lineChart4Up(range(wordsUsed),
allRCumulDiffLineData[d["name"]],
True,
sortedSaveDir,
"4up-r-cumul",
yLim=None,
yAdjust=1)
# Create graph charts for all data in a cloud
graphTypes = [
("all-diffs", allDiffLineData, None, 0),
("all-diffs-cumul", allCumulDiffLineData, None, 1),
#("all-pcts", allPercentageLineData, [0, 100], 0)
]
alls = {}
for graphType, lineList, yLim, adjust in graphTypes:
medFilename = baseSaveDir + "dists/median-%s.json" % graphType
med = utils.getContent(medFilename, True)
alls[graphType] = {}
for d in med:
lineList[d["name"]].append(["Median", d["line"]])
alls[graphType][d["name"]] = d["all"]
for name in allPercentageLineData:
sortedSaveDir = baseSaveDir + "wordImportance/" + name + "-sorted/"
for log in [False]: #, True]:
print(" %s..." % graphType)
graphUtils.lineChart(range(wordsUsed),
lineList[name],
True,
sortedSaveDir,
graphType,
yLim=yLim,
log=log,
yAdjust=adjust)
print(" %s cloud..." % graphType)
graphUtils.lineChart(range(wordsUsed),
lineList[name],
True,
sortedSaveDir,
graphType + "-cloud",
yLim=yLim,
allLines=alls[graphType][name],
log=log,
yAdjust=adjust)
# Create chart showing ignored top words
n = "Jensen-shannon"
sortedSaveDir = baseSaveDir + "wordImportance/" + n + "-sorted/"
# Cumulative
data = allCumulDiffLineData[n]
# Add lines
res = []
targetSim = -1
for item in alls["all-diffs-cumul"][n]:
name, c = item
# "Aristotle_Pindar" in name or
#"AeliusAristides_Demosthenes", "DioChrysostom_Plato"
if ("ApolloniusRhodius_QuintusSmyrnaeus" in name
or "DioChrysostom_Xenophon" == name):
res.append((name, "-", 1 + c[-1] - np.array(c)))
# Lowest of our top authors
if ("DioChrysostom_Xenophon" == name):
targetSim = c[-1]
# add median
# for item in allCumulDiffLineData[n]:
# name, c = item
# if ("Median" in name):
# res.append((name, "-", 1 + c[-1] - np.array(c)))
# Add line cloud
resAll = []
for item in alls["all-diffs-cumul"][n]:
name, c = item
if not ("Hymns_Dionysus" in name or "Euclid" in name):
n1, n2 = name.replace("Hymns_", "Hymns").split("_")
n1 = n1.replace("Hymns", "Hymns_")
n2 = n2.replace("Hymns", "Hymns_")
centuryDiff = centDiff(genre.toCent(n1), genre.toCent(n2))
#print("%s, %s: %d" % (n1, n2, centuryDiff))
if (centuryDiff >= 4):
# color top sims differently
color = "k-"
resAll.append((name, color, 1 + c[-1] - np.array(c)))
# for name, c in data:
# y = c[-1] - np.array(c)
# res.append((name, y))
#resAll = map(lambda n, c: (n, c[-1] - np.array(c)))
graphUtils.compareWordUsageChart(res,
True,
sortedSaveDir,
"ignoreBestWords",
yLim=None,
allLines=resAll)
import xml.dom.minidom, time, re, select, nntp, socket
import os, sys, string, ConfigParser, codecs, utils, time
import downloader, verifier, extractor, bookmarks, nntplib
# check nzb files
nfiles = []
if len(sys.argv) > 1:
for arg in sys.argv[1:]:
if utils.fileExists(arg):
if arg.split(".")[-1].lower() == "nzb":
nfiles.append({"path": arg, "isBookmark": False})
if arg[0] == "/":
# unix style
print "added [%s] to the queue" % string.split(arg, "/")[-1]
else:
# win32 style
print "added [%s] to the queue" % string.split(arg, "\\")[-1]
else:
print "not an nzb file: %s" % arg
# change the cwd
# NOTE: with py2exe use sys.executable instead of __file__
approot = os.path.dirname(__file__)
if approot != '':
os.chdir(approot)
# load settings
cfg = ConfigParser.ConfigParser()
cfg.read("settings.conf")
# load gui
