def checkNamespace(self, areaName):
bigMessage = []
noErrors = 0
pointCount = 0
bigMessage.append("ooooooooooooooooooooooooooooooooooooooooooooooooooo")
bigMessage.append(" NAMESPACE: " + areaName)
bigMessage.append("ooooooooooooooooooooooooooooooooooooooooooooooooooo")
namespace_manager.set_namespace(areaName)
# Take every point version
query = Point.query()
for point in query.iter():
foundError, newMessages = self.checkPoint(point)
bigMessage = bigMessage + newMessages
if not foundError:
noErrors = noErrors + 1
pointCount = pointCount + 1
bigMessage.append( "%d points checked. No errors detected in %d points" % (pointCount, noErrors))
noErrors = 0
rootCount = 0
query = PointRoot.query()
for pointRoot in query.iter():
foundError, newMessages = self.checkRoot(pointRoot)
bigMessage = bigMessage + newMessages
if not foundError:
noErrors = noErrors + 1
rootCount = rootCount + 1
bigMessage.append( "%d roots checked. No errors detected in %d roots" % (rootCount, noErrors))
return bigMessage
def get(self):
query = Point.query()
i = 0
for point in query.iter():
if point.supportingPoints:
for pointKey in point.supportingPoints:
point.supportingPointsRoots.append(pointKey)
point.supportingPoints = []
for rootKey in point.supportingPointsRoots:
root = rootKey.get()
if root:
pointVer = root.getCurrent()
point.supportingPointsLastChange.append(pointVer.key)
else:
logging.info('ROOTKEY %s WAS NOT FOUND' % rootKey)
else:
point.supportingPointsRoots = []
point.supportingPointsLastChange = []
point.put()
logging.info('Updating %s' % point.title)
i = i + 1
template_values = {
'message': "Edits made: %d" % i
}
path = os.path.join(os.path.dirname(__file__), '../templates/message.html')
self.response.out.write(template.render(path, template_values))
def getPointCreator(self):
result = {'result': False}
point, pointRoot = Point.getCurrentByUrl(self.request.get('pointURL'))
versionsOfThisPoint = Point.query(ancestor=pointRoot.key).order(Point.version)
firstVersion = versionsOfThisPoint.get()
authors = []
"""code for listing number of contributors"""
"""
for point in versionsOfThisPoint:
thisAuthor = {"authorName": point.authorName, "authorURL": point.authorURL }
if thisAuthor not in authors:
authors.append(thisAuthor)
"""
resultJSON = json.dumps({
'result': True,
'creatorName' : firstVersion.authorName,
'creatorURL' : firstVersion.authorURL,
'numAuthors' : len(authors)
})
self.response.headers["Content-Type"] = 'application/json; charset=utf-8'
self.response.out.write(resultJSON)
def checkNamespace(self, areaName):
bigMessage = []
noErrors = 0
pointCount = 0
bigMessage.append("ooooooooooooooooooooooooooooooooooooooooooooooooooo")
bigMessage.append(" NAMESPACE: " + areaName)
bigMessage.append("ooooooooooooooooooooooooooooooooooooooooooooooooooo")
namespace_manager.set_namespace(areaName)
# Take every point version
query = Point.query()
for point in query.iter():
foundError, newMessages = self.checkPoint(point)
bigMessage = bigMessage + newMessages
if not foundError:
noErrors = noErrors + 1
pointCount = pointCount + 1
bigMessage.append( "%d points checked. No errors detected in %d points" % (pointCount, noErrors))
noErrors = 0
rootCount = 0
query = PointRoot.query()
for pointRoot in query.iter():
foundError, newMessages = self.checkRoot(pointRoot)
bigMessage = bigMessage + newMessages
if not foundError:
noErrors = noErrors + 1
rootCount = rootCount + 1
bigMessage.append( "%d roots checked. No errors detected in %d roots" % (rootCount, noErrors))
return bigMessage
def get(self):
query = Point.query()
i = 0
for point in query.iter():
if point.supportingPoints:
for pointKey in point.supportingPoints:
point.supportingPointsRoots.append(pointKey)
point.supportingPoints = []
for rootKey in point.supportingPointsRoots:
root = rootKey.get()
if root:
pointVer = root.getCurrent()
point.supportingPointsLastChange.append(pointVer.key)
else:
logging.info('ROOTKEY %s WAS NOT FOUND' % rootKey)
else:
point.supportingPointsRoots = []
point.supportingPointsLastChange = []
point.put()
logging.info('Updating %s' % point.title)
i = i + 1
template_values = {'message': "Edits made: %d" % i}
path = os.path.join(os.path.dirname(__file__),
'../templates/message.html')
self.response.out.write(template.render(path, template_values))
def getPointCreator(self):
result = {'result': False}
point, pointRoot = Point.getCurrentByUrl(self.request.get('pointURL'))
versionsOfThisPoint = Point.query(ancestor=pointRoot.key).order(
Point.version)
firstVersion = versionsOfThisPoint.get()
authors = []
"""code for listing number of contributors"""
"""
for point in versionsOfThisPoint:
thisAuthor = {"authorName": point.authorName, "authorURL": point.authorURL }
if thisAuthor not in authors:
authors.append(thisAuthor)
"""
resultJSON = json.dumps({
'result': True,
'creatorName': firstVersion.authorName,
'creatorURL': firstVersion.authorURL,
'numAuthors': len(authors)
})
self.response.headers[
"Content-Type"] = 'application/json; charset=utf-8'
self.response.out.write(resultJSON)
def indClearLowQualityFlags(cursor=None, num_updated=0, batch_size=250, cntUpdatedNet=0, namespace=None, namespaces=None):
logging.info('ClearLowQuality Update: Start: %d Batch: %d Namespace: %s' % (num_updated, batch_size, namespace))
if namespace:
previous_namespace = namespace_manager.get_namespace()
namespace_manager.set_namespace(namespace)
else:
previous_namespace = None
try:
query = Point.query()
points, next_cursor, more = query.fetch_page(batch_size, start_cursor=cursor)
cnt = 0
cntSkip = 0
cntUpdate = 0
# for p in query.iter():
for p in points:
cnt += 1
# if p.isLowQualityAdmin == False:
# cntSkip += 1
# continue
if p.isLowQualityAdmin:
cntUpdate += 1
p.isLowQualityAdmin = False
p.put()
logging.info('ClearLowQuality Incremental: Count: %d Updated: %d Skipped: %d' % (cnt, cntUpdate, cntSkip))
finally:
if previous_namespace:
namespace_manager.set_namespace(previous_namespace)
# If there are more entities, re-queue this task for the next page.
if more:
deferred.defer(indClearLowQualityFlags,
cursor=next_cursor,
num_updated=(num_updated + cnt),
batch_size=batch_size,
cntUpdatedNet=(cntUpdatedNet + cntUpdate),
namespace=namespace,
namespaces=namespaces)
else:
logging.warning('ClearLowQuality Complete! - Net Updated: %d Namespace: %s' % (cntUpdatedNet + cntUpdate, namespace))
if namespaces and len(namespaces) > 0:
nextNamespace = namespaces[0]
del namespaces[0]
logging.warning('ClearLowQuality: Next Namespace: %s Count: %d' % (nextNamespace, len(namespaces)))
deferred.defer(indClearLowQualityFlags,
cursor=None,
num_updated=0,
batch_size=batch_size,
cntUpdatedNet=0,
namespace=nextNamespace,
namespaces=namespaces)
def changeUserUrl(cursor=None, num_updated=0, batch_size=250, cntUpdatedNet=0, namespace=None, namespaces=None):
logging.info('ChangeUserUrl Update: Start: %d Batch: %d Namespace: %s' % (num_updated, batch_size, namespace))
query = Point.query()
points, next_cursor, more = query.fetch_page(batch_size, start_cursor=cursor)
cnt = 0
cntSkip = 0
cntUpdate = 0
# for p in query.iter():
for p in points:
cnt += 1
if p.authorURL != 'Tom_Gratian' and p.authorURL != 'tom_gratian' and p.creatorURL != 'Tom_Gratian' and p.creatorURL != 'tom_gratian':
continue
logging.warning('ChangeUserUrl: Update: %s Author -> (%s, %s)' % (p.url, p.authorName, p.authorURL))
p.authorName = 'Big T'
p.creatorName = 'Big T'
p.put()
cntUpdate += 1
# p.isLowQualityAdmin = False
# p.put()
logging.info('ChangeUserUrl Incremental: Count: %d Updated: %d Skipped: %d' % (cnt, cntUpdate, cntSkip))
# If there are more entities, re-queue this task for the next page.
if more:
deferred.defer(changeUserUrl,
cursor=next_cursor,
num_updated=(num_updated + cnt),
batch_size=batch_size,
cntUpdatedNet=(cntUpdatedNet + cntUpdate),
namespace=namespace,
namespaces=namespaces)
else:
logging.warning('ChangeUserUrl Complete! - Net Updated: %d Namespace: %s' % (cntUpdatedNet + cntUpdate, namespace))
def callForAllPoints(pointUpdate, pointIsEligible=None,
cursor=None, num_updated=0, batch_size=250, cntUpdatedNet=0,
namespace=None, namespaces=None):
logging.info('CallForAllPoints: Start: %d Batch: %d Namespace: %s' % (num_updated, batch_size, namespace))
if namespace:
previous_namespace = namespace_manager.get_namespace()
namespace_manager.set_namespace(namespace)
else:
previous_namespace = None
try:
query = Point.query()
points, next_cursor, more = query.fetch_page(batch_size, start_cursor=cursor)
cnt = 0
cntUpdate = 0
# for p in query.iter():
for p in points:
cnt += 1
if pointIsEligible:
if not (pointIsEligible(p)):
continue
pointUpdate(p)
cntUpdate += 1
finally:
if previous_namespace:
namespace_manager.set_namespace(previous_namespace)
# If there are more entities, re-queue this task for the next page.
if more:
logging.info('CallForAllPoints Incremental: Count: %d Updated: %d' % (cnt, cntUpdate))
deferred.defer(callForAllPoints,
pointUpdate=pointUpdate,
pointIsEligible=pointIsEligible,
cursor=next_cursor,
num_updated=(num_updated + cnt),
batch_size=batch_size,
cntUpdatedNet=(cntUpdatedNet + cntUpdate),
namespace=namespace,
namespaces=namespaces)
else:
logging.info('CallForAllPoints Complete! - Net Updated: %d' % (cntUpdatedNet + cntUpdate))
if namespaces and len(namespaces) > 0:
nextNamespace = namespaces[0]
del namespaces[0]
logging.warning('CallForAllPoints: Next Namespace: %s Count: %d' % (nextNamespace, len(namespaces)))
deferred.defer(callForAllPoints,
pointUpdate=pointUpdate,
pointIsEligible=pointIsEligible,
cursor=None,
num_updated=0,
batch_size=batch_size,
cntUpdatedNet=0,
namespace=nextNamespace,
namespaces=namespaces)
else:
logging.warning('CallForAllPoints Final Complete! - Net Updated: %d' % (cntUpdatedNet + cntUpdate))
def checkRoot(self, pointRoot):
try:
foundError = False
messages = []
# linkVal = pointRoot.key.urlsafe()
# bigMessage.append("-------- Checking Root %s: <a href=\"%s%s\">%s</a> "\
# % (pointRoot.url, constants.ADMIN_DATA_URL, linkVal, pointRoot.key))
curPoint = pointRoot.getCurrent()
if not curPoint:
messages.append("Not able to get current from <a href=\"/point/%s\">%s</a>. " % \
(pointRoot.url, pointRoot.url))
foundError = True
elif not curPoint.current:
messages.append("Root <a href=\"/point/%s\">%s</a>: \
Current point is not marked current" % \
(pointRoot.url, curPoint.title))
foundError = True
pointQuery = Point.query(ancestor=pointRoot.key)
points = pointQuery.fetch()
curCount = 0
curURLs = ""
for point in points:
if point.current:
curCount = curCount + 1
curURLs = curURLs + point.key.urlsafe()+ ","
if curCount != 1:
messages.append("Root <a href=\"/point/%s\">%s</a>: \
Found %d points marked current. URL keys: %s" % \
(pointRoot.url, pointRoot.url, \
curCount, curURLs))
messages.append('<a href=\"/job/cleanCurrents/%s\"> \
Clean multiple current points</a>' % pointRoot.url)
foundError = True
for linkType in ["supporting", "counter"]:
linkPoints, archivedLinkPoints = \
pointRoot.getBacklinkCollections(linkType)
for linkRootKey in linkPoints:
linkRoot = linkRootKey.get()
if not linkRoot:
messages.append("Root <a href=\"/point/%s\">%s</a>: \
Not able to get %s backlink root by link root key %s" % \
( pointRoot.url, pointRoot.url, \
linkType, linkRootKey))
foundError = True
continue
currentLinkPoint = linkRoot.getCurrent()
linkedPoints = currentLinkPoint.getLinkedPointsRootKeys(linkType)
if not pointRoot.key in linkedPoints:
versionKeyURL = currentLinkPoint.key.urlsafe()
messages.append("Root <a href=\"/point/%s\">%s</a>: \
Have %s backlink to ' \
<a href=\"%s%s\">%s</a> but no link root." % \
( pointRoot.url, pointRoot.url,\
linkType, currentLinkPoint.url, \
currentLinkPoint.title))
foundError = True
return foundError, messages
except Exception as e:
errMsg = 'Exception %s when checking %s' % (pointRoot.url, str(e))
return True, messages + [errMsg]
def checkNamespace(self, areaName):
bigMessage = []
noErrors = 0
pointCount = 0
bigMessage.append("ooooooooooooooooooooooooooooooooooooooooooooooooooo")
bigMessage.append(" NAMESPACE: " + areaName)
bigMessage.append("ooooooooooooooooooooooooooooooooooooooooooooooooooo")
namespace_manager.set_namespace(areaName)
# Take every point version
query = Point.query()
for point in query.iter():
foundError = False
checkingPointURL = point.key.urlsafe()
# bigMessage.append("-------- Checking Point %s: <a href=\"%s%s\">%s</a> "\
# % (point.title, constants.ADMIN_DATA_URL, checkingPointURL, point.key))
for linkType in ["supporting", "counter"]:
linkRoots, linkLastChange = point.getLinkCollections(linkType)
if linkLastChange:
for pointKey in linkLastChange:
# Check based on the version array
# 1. Every linked point in the version array
# has to have a root in the root array
if not pointKey.parent() in linkRoots:
bigMessage.append( \
"Point %s. Version %d: Missing corresponding root \
for %s point %s " % (point.url, point.version, linkType, str(pointKey)))
foundError = True
if point.current:
# 2. Every linked point in the link array of a current point
# should have backlinks in the root of the linked point
linkRoot = pointKey.parent().get()
backlinks, archiveBacklinks = linkRoot.getBacklinkCollections(linkType)
if not point.key.parent() in backlinks:
linkedPointURL = linkRoot.key.urlsafe()
bigMessage.append(
"Point %s. Version %d: Has %s link to \
<a href=\"/point/%s\">%s</a> with no BACKLINK" \
% (point.url, point.version, linkType, \
linkRoot.url, linkRoot.url))
foundError = True
if len(linkLastChange) != len(linkRoots):
bigMessage.append(
"Point: <a href=\"/point/%s\">%s</a>. Version %d: \
Length mismatch in %s arrays. \
Version List has: %d. Root Array has: %d " % \
(point.url, point.title, point.version,
linkType, len(linkLastChange), \
len(linkRoots)))
foundError = True
if not foundError:
noErrors = noErrors + 1
pointCount = pointCount + 1
bigMessage.append( "%d points checked. No errors detected in %d points" % (pointCount, noErrors))
noErrors = 0
rootCount = 0
query = PointRoot.query()
for pointRoot in query.iter():
foundError = False
linkVal = pointRoot.key.urlsafe()
# bigMessage.append("-------- Checking Root %s: <a href=\"%s%s\">%s</a> "\
# % (pointRoot.url, constants.ADMIN_DATA_URL, linkVal, pointRoot.key))
curPoint = pointRoot.getCurrent()
if not curPoint.current:
bigMessage.append("Root <a href=\"/point/%s\">%s</a>: \
Current point is not marked current" % \
(pointRoot.url, curPoint.title))
foundError = True
pointQuery = Point.query(ancestor=pointRoot.key)
points = pointQuery.fetch()
curCount = 0
curURLs = ""
for point in points:
if point.current:
curCount = curCount + 1
curURLs = curURLs + point.key.urlsafe()+ ","
if curCount != 1:
bigMessage.append("Root <a href=\"/point/%s\">%s</a>: \
Found %d points marked current. URL keys: %s" % \
(pointRoot.url, pointRoot.url, \
curCount, curURLs))
foundError = True
for linkType in ["supporting", "counter"]:
linkPoints, archivedLinkPoints = \
pointRoot.getBacklinkCollections(linkType)
for linkRootKey in linkPoints:
linkRoot = linkRootKey.get()
if not linkRoot:
bigMessage.append("Root <a href=\"/point/%s\">%s</a>: \
Not able to get %s backlink root by link root key %s" % \
( pointRoot.url, pointRoot.url, \
linkType, linkRootKey))
foundError = True
continue
currentLinkPoint = linkRoot.getCurrent()
linkedPoints = currentLinkPoint.getLinkedPointsRootCollection(linkType)
if not pointRoot.key in linkedPoints:
versionKeyURL = currentLinkPoint.key.urlsafe()
bigMessage.append("Root <a href=\"/point/%s\">%s</a>: \
Have %s backlink to ' \
<a href=\"%s%s\">%s</a> but no link root." % \
( pointRoot.url, pointRoot.url,\
linkType, currentLinkPoint.url, \
currentLinkPoint.title))
foundError = True
if not foundError:
noErrors = noErrors + 1
rootCount = rootCount + 1
bigMessage.append( "%d roots checked. No errors detected in %d roots" % (rootCount, noErrors))
return bigMessage
def checkRoot(self, pointRoot):
try:
foundError = False
messages = []
# linkVal = pointRoot.key.urlsafe()
# bigMessage.append("-------- Checking Root %s: <a href=\"%s%s\">%s</a> "\
# % (pointRoot.url, constants.ADMIN_DATA_URL, linkVal, pointRoot.key))
curPoint = pointRoot.getCurrent()
if not curPoint:
messages.append("Not able to get current from <a href=\"/point/%s\">%s</a>. " % \
(pointRoot.url, pointRoot.url))
foundError = True
elif not curPoint.current:
messages.append("Root <a href=\"/point/%s\">%s</a>: \
Current point is not marked current" % \
(pointRoot.url, curPoint.title))
foundError = True
pointQuery = Point.query(ancestor=pointRoot.key)
points = pointQuery.fetch()
curCount = 0
curURLs = ""
for point in points:
if point.current:
curCount = curCount + 1
curURLs = curURLs + point.key.urlsafe() + ","
if curCount != 1:
messages.append("Root <a href=\"/point/%s\">%s</a>: \
Found %d points marked current. URL keys: %s" % \
(pointRoot.url, pointRoot.url, \
curCount, curURLs))
messages.append('<a href=\"/job/cleanCurrents/%s\"> \
Clean multiple current points</a>' % pointRoot.url)
foundError = True
for linkType in ["supporting", "counter"]:
linkPoints, archivedLinkPoints = \
pointRoot.getBacklinkCollections(linkType)
for linkRootKey in linkPoints:
linkRoot = linkRootKey.get()
if not linkRoot:
messages.append("Root <a href=\"/point/%s\">%s</a>: \
Not able to get %s backlink root by link root key %s" % \
( pointRoot.url, pointRoot.url, \
linkType, linkRootKey))
foundError = True
continue
currentLinkPoint = linkRoot.getCurrent()
linkedPoints = currentLinkPoint.getLinkedPointsRootKeys(
linkType)
if not pointRoot.key in linkedPoints:
versionKeyURL = currentLinkPoint.key.urlsafe()
messages.append("Root <a href=\"/point/%s\">%s</a>: \
Have %s backlink to ' \
<a href=\"%s%s\">%s</a> but no link root." % \
( pointRoot.url, pointRoot.url,\
linkType, currentLinkPoint.url, \
currentLinkPoint.title))
foundError = True
return foundError, messages
except Exception as e:
errMsg = 'Exception %s when checking %s' % (pointRoot.url, str(e))
return True, messages + [errMsg]
def resolve_points(self, info, **args):
return PointModel.query()
