def setDumpParsed(dump_id):
cursor.execute(
'''UPDATE dumpinfo SET parsed = True
WHERE id = %s''', (dump_id, ))
connection.commit()
return True
def addResource(content_id,
entitytype,
value,
is_banned=None,
last_change=None,
atomic=False):
"""
Not atomic
Adds resource to the table
:return: new resource ID
"""
cursor.execute(
'''INSERT INTO resource
(content_id, last_change, entitytype_id, value, is_banned)
VALUES (%s, %s,
(SELECT id FROM entitytype WHERE name = %s),
%s, %s)
RETURNING id
''', (
content_id,
last_change,
entitytype,
value,
is_banned,
))
if atomic:
connection.commit()
return cursor.fetchone()['id']
def updateDumpCheckTime():
now = datetime.now().astimezone()
cursor.execute(
'''UPDATE dumpinfo SET check_time = %s
WHERE id = (SELECT MAX(id) FROM dumpinfo)
AND parsed = True
''', (now, ))
connection.commit()
return True
def finishJob(log_id, exit_code, result=None):
cursor.execute(
'UPDATE log SET exit_code=%s, finish_time=%s, result=%s WHERE id=%s', (
exit_code,
datetime.now().astimezone(),
result,
log_id,
))
connection.commit()
def unlockJobs(procname=None):
query = sql.SQL('''UPDATE log SET exit_code=%s
WHERE exit_code is Null''')
if procname:
query = query + sql.SQL(' AND procname = {0}').format(
sql.Literal(procname))
cursor.execute(query, (UNLOCK_EXIT_CODE, ))
connection.commit()
return int(cursor.statusmessage.split(' ')[1])
def addDumpInfoRecord(updateTime, updateTimeUrgently, **kwargs):
"""
The arguments were named corresponding with
<reg> tag attributes to simplify kwargs passthrough
"""
now = datetime.now().astimezone()
cursor.execute(
'''INSERT INTO dumpinfo
(update_time, update_time_urgently, parse_time, check_time, parsed)
VALUES (%s, %s, %s, %s, %s) RETURNING id''',
(updateTime, updateTimeUrgently, now, now, False))
connection.commit()
return cursor.fetchone()['id']
def delContent(outer_id):
"""
Deletes content and all its resources
:param outer_id: from dump
:return: content_id, or None if nothing deleted
"""
cursor.execute(
'''DELETE FROM content
WHERE outer_id = %s
RETURNING id
''', (outer_id, ))
connection.commit()
if cursor.rowcount > 0:
return cursor.fetchone()['id']
else:
return None
def updateContentPresence(dump_id, disabledIDList=[]):
"""
Sets in_dump to false for removed IDs from the set
Sets the latest dump_id label from alive records
:param dump_id: dump id
:param disabledIDList: IDs list
:return: True
"""
cursor.execute(
'''UPDATE content SET in_dump = False
WHERE outer_id = ANY(%s)''', (disabledIDList, ))
cursor.execute(
'''UPDATE content SET last_dump_id = %s
WHERE in_dump = True''', (dump_id, ))
connection.commit()
return True
def delCustomResource(entitytype, value):
"""
Deletes custom resource from the table.
:return: row ID or None
"""
cursor.execute(
'''DELETE FROM resource
WHERE is_banned IS NOT NULL
AND value = %s
AND entitytype_id = (SELECT id FROM entitytype WHERE name = %s)
RETURNING id
''', (
value,
entitytype,
))
connection.commit()
if cursor.rowcount > 0:
return cursor.fetchone()['id']
return None
def addCustomResource(entitytype, value, is_banned=False):
"""
Adds custom resource to the table.
:return: new or existing resource ID
"""
now = datetime.now().astimezone()
cursor.execute(
'''SELECT resource.id FROM resource
JOIN entitytype ON resource.entitytype_id = entitytype.id
WHERE is_banned IS NOT NULL
AND entitytype.name = %s
AND value=%s
''', (
entitytype,
value,
))
# If exists, banning/unbanning with returning IDs
if cursor.rowcount > 0:
ids = [c['id'] for c in cursor]
cursor.execute(
'''
UPDATE resource
SET is_banned = %s, last_change = %s
WHERE ID=ANY(%s)
''', (
is_banned,
now,
ids,
))
connection.commit()
return ids
# If nothing found
return addResource(content_id=None,
entitytype=entitytype,
value=value,
is_banned=is_banned,
last_change=now,
atomic=True)
def addDecision(date, number, org, atomic=False):
"""
The arguments were named corresponding with
<decision> tag attributes to simplify kwargs passthrough
:return: decision ID
"""
cursor.execute(
'''SELECT id FROM decision
WHERE decision_code = %s''', (number, ))
# Decision exists in the database
if cursor.rowcount > 0:
return cursor.fetchone()['id']
# Adding organisation to the table if missing
cursor.execute('''SELECT id FROM organisation
WHERE name = %s''', (org, ))
if cursor.rowcount == 0:
cursor.execute(
'''INSERT INTO organisation (name)
VALUES (%s) RETURNING id''',
(org, ),
)
org_id = cursor.fetchone()['id']
# Adding missing decision to the table
cursor.execute(
'''INSERT INTO DECISION
(decision_code, decision_date, org_id)
VALUES (%s, %s, %s) RETURNING id''', (
number,
date,
org_id,
))
if atomic:
connection.commit()
return cursor.fetchone()['id']
def addContent(dump_id,
decision_id,
id,
includeTime,
hash,
entryType,
blockType='default',
ts=None,
atomic=False,
**kwargs):
"""
Not atomic
The arguments were named corresponding with
<content> tag attributes to simplify kwargs passthrough
"""
# Checking whether content is in the table but disabled
# Cascade purging must place to be
cursor.execute(
'''DELETE FROM content
WHERE outer_id = %s
AND in_dump is False
''', (id, ))
cursor.execute(
'''INSERT INTO content
VALUES (DEFAULT, %s, %s, %s, %s, %s, %s,
(SELECT id FROM blocktype WHERE name = %s),
%s, %s, %s)
RETURNING id
''', (id, includeTime, hash, ts, True, decision_id, blockType,
entryType, dump_id, dump_id))
if atomic:
connection.commit()
return cursor.fetchone()['id']
def commitChanges():
# Force committing after long transaction
connection.commit()
def addLogEntry(procname):
cursor.execute(
'INSERT INTO log (start_time,procname) VALUES (%s,%s) RETURNING id',
(datetime.now(), procname))
connection.commit()
return cursor.fetchone()['id']