def removeOldUsers(months=4):
cursor = get_db().cursor()
executeQuery(
cursor,
'DELETE FROM #PFX#users WHERE ADDDATE(mtime, INTERVAL %s MONTH) < NOW()',
months)
get_db().commit()
def removeOldReports(days=30):
cursor = get_db().cursor(MySQLdb.cursors.DictCursor)
executeQuery(cursor,
'''SELECT guid FROM #PFX#reports WHERE ADDDATE(ctime, INTERVAL %s DAY) < NOW()''',
(days))
for report in cursor:
removeReport(report['guid'])
def processReport(xmlFile):
global reportData, tagStack
guid = os.path.splitext(os.path.basename(xmlFile))[0]
cursor = get_db().cursor()
executeQuery(cursor,
'''SELECT guid FROM #PFX#reports WHERE guid = %s''',
(guid))
report = cursor.fetchone()
if report != None:
os.remove(xmlFile)
return
source = open(xmlFile, 'rb')
reportData = {'status': '', 'usefulness': 0, 'warnings': {}, 'requests': [], 'filters': [], 'subscriptions': [], 'extensions': [], 'errors': [], 'time': time()}
tagStack = []
parser = ParserCreate()
parser.StartElementHandler = processElementStart
parser.EndElementHandler = processElementEnd
parser.CharacterDataHandler = processText
try:
parser.ParseFile(source)
except ExpatError, error:
reportData['warnings']['!parsing'] = 'Parsing error in the report: %s at line %i column %i' % (ErrorString(error.code), error.lineno, error.offset)
def processReport(xmlFile):
global reportData, tagStack
guid = os.path.splitext(os.path.basename(xmlFile))[0]
cursor = get_db().cursor()
executeQuery(cursor,
'''SELECT guid FROM #PFX#reports WHERE guid = %s''',
(guid))
report = cursor.fetchone()
if report != None:
os.remove(xmlFile)
return
source = open(xmlFile, 'rb')
reportData = {'status': '', 'usefulness': 0, 'warnings': {}, 'requests': [], 'filters': [], 'subscriptions': [], 'extensions': [], 'errors': [], 'time': time()}
tagStack = []
parser = ParserCreate()
parser.StartElementHandler = processElementStart
parser.EndElementHandler = processElementEnd
parser.CharacterDataHandler = processText
try:
parser.ParseFile(source)
except ExpatError, error:
reportData['warnings']['!parsing'] = 'Parsing error in the report: %s at line %i column %i' % (ErrorString(error.code), error.lineno, error.offset)
def removeOldReports(days=30):
cursor = get_db().cursor(MySQLdb.cursors.DictCursor)
executeQuery(
cursor,
'SELECT guid FROM #PFX#reports WHERE ADDDATE(ctime, INTERVAL %s DAY) < NOW()',
days)
for report in cursor:
removeReport(report['guid'])
def updateSubscriptionList():
cursor = get_db().cursor(MySQLdb.cursors.DictCursor)
executeQuery(cursor, 'SELECT id, url FROM #PFX#subscriptions')
subids = {}
for dbsub in cursor:
subids[dbsub['url']] = dbsub['id']
subscriptions = subscriptionParser.readSubscriptions()
for subscription in subscriptions.values():
for title, url, complete in subscription.variants:
id = subids.get(url)
if id == None:
executeQuery(cursor, 'INSERT INTO #PFX#subscriptions (url) VALUES (%s)', url)
else:
del subids[url]
for url in subids:
executeQuery(cursor, 'DELETE FROM #PFX#subscriptions WHERE id = %s', subids[url])
get_db().commit()
def processReport(xmlFile):
global reportData, tagStack
guid = os.path.splitext(os.path.basename(xmlFile))[0]
cursor = get_db().cursor()
executeQuery(cursor, 'SELECT guid FROM #PFX#reports WHERE guid = %s', guid)
report = cursor.fetchone()
if report != None:
os.remove(xmlFile)
return
source = open(xmlFile, 'rb')
reportData = {
'status': '',
'usefulness': 0,
'warnings': {},
'requests': [],
'filters': [],
'subscriptions': [],
'extensions': [],
'errors': [],
'time': time()
}
tagStack = []
parser = ParserCreate()
parser.StartElementHandler = processElementStart
parser.EndElementHandler = processElementEnd
parser.CharacterDataHandler = processText
try:
parser.ParseFile(source)
except ExpatError as error:
reportData['warnings'][
'!parsing'] = 'Parsing error in the report: %s at line %i column %i' % (
ErrorString(error.code), error.lineno, error.offset)
source.seek(0)
reportData['knownIssues'] = knownIssuesParser.findMatches(source, 'en-US')
source.close()
if 'screenshot' in reportData and not reportData['screenshot'].startswith(
'data:image/'):
del reportData['screenshot']
if 'email' in reportData and reportData['email'].find(
' at ') < 0 and reportData['email'].find('@') < 0:
del reportData['email']
validateData(reportData)
saveReport(guid, reportData, True)
os.remove(xmlFile)
def getReports():
count = 1000
offset = 0
while True:
cursor = get_db().cursor(MySQLdb.cursors.DictCursor)
executeQuery(cursor, 'SELECT guid FROM #PFX#reports WHERE hasscreenshot > 0 LIMIT %s OFFSET %s', (count, offset))
rows = cursor.fetchall()
cursor.close()
if len(rows) == 0:
break
for row in rows:
yield row
offset += len(rows)
def getReports():
count = 1000
offset = 0
while True:
cursor = get_db().cursor(MySQLdb.cursors.DictCursor)
executeQuery(
cursor,
'SELECT guid FROM #PFX#reports WHERE hasscreenshot > 0 LIMIT %s OFFSET %s',
(count, offset))
rows = cursor.fetchall()
cursor.close()
if len(rows) == 0:
break
for row in rows:
yield row
offset += len(rows)
def processReport(xmlFile):
global reportData, tagStack
guid = os.path.splitext(os.path.basename(xmlFile))[0]
cursor = get_db().cursor()
executeQuery(cursor, 'SELECT guid FROM #PFX#reports WHERE guid = %s', guid)
report = cursor.fetchone()
if report != None:
os.remove(xmlFile)
return
source = open(xmlFile, 'rb')
reportData = {'status': '', 'usefulness': 0, 'warnings': {}, 'requests': [], 'filters': [], 'subscriptions': [], 'extensions': [], 'errors': [], 'time': time()}
tagStack = []
parser = ParserCreate()
parser.StartElementHandler = processElementStart
parser.EndElementHandler = processElementEnd
parser.CharacterDataHandler = processText
try:
parser.ParseFile(source)
except ExpatError as error:
reportData['warnings']['!parsing'] = 'Parsing error in the report: %s at line %i column %i' % (ErrorString(error.code), error.lineno, error.offset)
source.seek(0)
reportData['knownIssues'] = knownIssuesParser.findMatches(source, 'en-US')
source.close()
if 'screenshot' in reportData and not reportData['screenshot'].startswith('data:image/'):
del reportData['screenshot']
if 'email' in reportData and reportData['email'].find(' at ') < 0 and reportData['email'].find('@') < 0:
del reportData['email']
validateData(reportData)
saveReport(guid, reportData, True)
os.remove(xmlFile)
def removeOldUsers(months=4):
cursor = get_db().cursor()
executeQuery(cursor, 'DELETE FROM #PFX#users WHERE ADDDATE(mtime, INTERVAL %s MONTH) < NOW()', months)
get_db().commit()