def insert(self, mysql: MySQLWrapper):
mysql.query(
"INSERT INTO Authors (HLTVID, Name) VALUES (%s, %s) ON DUPLICATE KEY UPDATE Name=%s;",
(self.hltvID, self.name, self.name,)
)
self.sqlID = self.getSQLID(mysql)
def getEndSignal(mysql: MySQLWrapper):
mysql.query("SELECT Value FROM Signals WHERE SignalName='End';")
results = mysql.fetchResults()
if results is None or results[0][0] == 1:
return True
else:
return False
def getRefreshTime(mysql: MySQLWrapper):
mysql.query("SELECT Value FROM Signals WHERE SignalName='Refresh';")
results = mysql.fetchResults()
if results is None:
return 30
else:
return results[0][0]
def insert(self, mysql: MySQLWrapper):
mysql.query(
(
"INSERT INTO Threads (HLTVID, ForumID, AuthorID, Title, NumResponses, Time) VALUES "
"(%s, %s, %s, %s, %s, %s) "
"ON DUPLICATE KEY UPDATE HLTVID=%s;"
),
(self.hltvID, self.forumID, self.author.sqlID, self.title, len(self.posts), self.timestamp, self.hltvID,)
)
self.sqlID = self.getSQLID(mysql)
def insert(self, mysql: MySQLWrapper):
mysql.query(
(
"INSERT INTO Posts (HLTVID, ThreadID, ReplyNum, AuthorID, Content, Time, HateRating, OffRating) VALUES "
"(%s, %s, %s, %s, %s, %s, %s, %s) "
"ON DUPLICATE KEY UPDATE HLTVID=%s;"
),
(self.getHLTVID(), self.threadID, self.index, self.author.sqlID, self.content, self.timestamp, self.hateRating, self.offRating, self.getHLTVID(),)
)
self.sqlID = self.getSQLID(mysql)
def getSQLID(self, mysql: MySQLWrapper):
mysql.query(
"SELECT ThreadID FROM Threads WHERE HLTVID=%s;",
(self.hltvID,)
)
result = mysql.fetchResults()
if result is None:
return None
else:
return result[0][0]
def getSQLID(self, mysql: MySQLWrapper):
mysql.query(
"SELECT PostID FROM Posts WHERE HLTVID=%s;",
(self.getHLTVID(),)
)
result = mysql.fetchResults()
if result is None:
return None
else:
return result[0][0]
def getForums(mysql: MySQLWrapper):
mysql.query("SELECT ForumID, HLTVID, Name FROM Forums;")
results = mysql.fetchResults()
if results is None:
return []
else:
forums = []
for row in results:
forumInfo = Forum(
sqlID = row[0], hltvID = row[1], name = row[2]
)
if '/' in forumInfo.hltvID:
forums += [forumInfo]
return forums
def setEndSignal(mysql: MySQLWrapper, enable: bool):
enableInt = 0
if enable:
enableInt = 1
mysql.query("INSERT INTO Signals (SignalName, Value) VALUES (%s, %s) ON DUPLICATE KEY UPDATE Value=%s;", ('End', enableInt, enableInt,))
mysql.db.commit()
overwrite = False
# Creates required tables
initializeTables(mysql, overwrite)
# Defines forums to be observed
forums = []
forums += [Forum("Offtopic", "17/off-topic")]
forums += [Forum("CSGO", "28/counter-strike-global-offensive")]
forums += [Forum("Hardware", "16/hardware-tweaks")]
for forum in forums:
forum.insert(mysql)
# By default, the forums are scraped every 15min
mysql.query("INSERT INTO Signals (SignalName, Value) VALUES ('Refresh', 15) ON DUPLICATE KEY UPDATE SignalName='Refresh';")
mysql.db.commit()
# Starts a new HTML-session
session = HTMLSession()
setEndSignal(mysql, False)
while not getEndSignal(mysql):
# Resets the refresh timer and debug output
lastUpdateTime = datetime.now()
timeStr = lastUpdateTime.strftime("%d.%m.%Y, %H:%M:%S")
print(f"Starting new update at {timeStr}!")
# Resets data counter each cycle
byteCount = 0
from mysqlwrapper import MySQLWrapper
from authorization import AuthorizationInfo
auth = AuthorizationInfo("auth.json")
mysql = MySQLWrapper(auth)
mysql.query("UPDATE Signals SET Active=1 WHERE SignalName=%s;", ('End'))
print(
'Sent end signal to scraper. It will terminate after the current cycle is complete.'
)