def databaseTester(self):
testReplay = '87B11D4743F1EAE859CAC9B48DAA8E89.replay'
# testDB = pd.read_csv(defaultDatabasePath)
# print(testDB)
dbm = DatabaseManager()
dbm.read()
#print(dbm.database)
row = dbm.getRow(testReplay)
print(row)
value = row.iloc[0]['GUID']
print(value)
class Uploader:
# Specifies the endpoints for the file uploads and replay data retriever
uploadURL = 'https://calculated.gg/api/upload'
replayURL = 'https://calculated.gg/api/replay/'
# Takes in the attributes from the file manager object
# Also instantiates:
# The status manager to continuously clean up the output interface
# The database manager to interface with the local replay database
def __init__(self, attributes):
self.attributes = attributes
self.replayList = attributes[1]
self.replayCount = attributes[2]
self.replayBatches = attributes[4]
self.batchAmount = attributes[5]
self.batchCount = attributes[6]
self.databaseManager = DatabaseManager()
self.statusUpdater = StatusUpdater()
self.replayDatabase = self.createDatabase()
self.statusUpdater.clearScreen()
# Uploads the replay files in batches:
# The uploader uses a progress bar to provide the user with the progress of the full upload
# Before uploading, the server is queried with the replay's unique identifier to ensure that it will not be a duplicate upload
# Once the batch is fully uploaded to the server, its parse status is continuously monitored
# After 60 seconds have passed or the batch has been fully parsed by the server, the next batch is uploaded
def upload(self):
for batch in tqdm(self.replayBatches,
desc='Upload progress: ',
position=0):
self.statusPayloads = []
self.previouslyUploaded = []
for replay in tqdm(batch,
desc=' Batch upload progress: ',
ncols=100,
position=1):
try:
self.postReplay(replay)
except Exception as e:
continue
try:
self.monitorBatchStatus(self.statusPayloads)
except Exception as e:
continue
print('\n\nAll files have been successfully uploaded!')
# Uploads a replay to the server
# Before the upload, the replay's unique identifier is retrieved and used to check for a duplicate upload
# The post request's reply is stored and added to the status list of the batch for use by the status monitor
def postReplay(self, replay):
currentGUID = self.getGUID(replay, self.replayDatabase)
if currentGUID is not None:
if self.checkForDuplicateUpload(currentGUID):
self.previouslyUploaded.append(True)
return
openReplay = {'replays': open(replay[0], 'rb')}
reply = self.getUploadReply(openReplay)
self.statusPayloads.append(self.getReplayPayload(reply, replay[1]))
# Queries the server for the replay's unique identifier
# If the replay already exists on the server, the response will contain data from the replay
# If the replay was not found on the server, the response will contain the word message
def checkForDuplicateUpload(self, replayGUID):
status = requests.get(self.replayURL + replayGUID)
response = list(status.json())
if 'message' in response:
return False
return True
# Sends the file to the server and returns the server's reply
# The post request is sent using the upload endpoint as well as the opened file from the postReplay function
def getUploadReply(self, replayFile):
return requests.post(self.uploadURL, files=replayFile)
# Uses the status code returned from the getUploadReply function to get the id of the replay stored on the server
# This id is used by the status monitor to query for the parse status without having the upload the file again
def getReplayPayload(self, reply, replayName):
if reply.status_code == 202:
reply_id = list(reply.json())[0]
payload = {'ids': reply_id}
fullPayload = [replayName, payload]
return fullPayload
# Status monitor for the parse status of a batch
# Takes in the payload list, containing the replay ids, of the uploaded files
# Checks for parse status for 60 seconds or until every replay returns a success or fail
def monitorBatchStatus(self, payloads, startTime=time.time()):
if not any(payloads) or all(True
for duplicate in self.previouslyUploaded):
self.statusUpdater.clearScreen()
return
self.statusUpdater.update(
self.statusUpdater.UploadState.waitingForSuccessfulUpload, '')
while True:
uploadStatus = self.getUploadBatchStatus(payloads)
if self.checkForBatchResponse(uploadStatus, 'SUCCESS'):
self.statusUpdater.update(
self.statusUpdater.UploadState.uploadSuccessful, '')
break
if self.checkForTimeout(startTime, 10):
self.statusUpdater.update(
self.statusUpdater.UploadState.uploadTimeout, '')
break
time.sleep(5)
self.delayAfterUpload(DELAY_AMOUNT)
self.statusUpdater.clearScreen()
# Queries the server for the parse status of every replay in the current batch
# The status of the batch is stored in a list with each element in the format:
# [Replay name, Status code]
def getUploadBatchStatus(self, statusPayloads):
statusList = []
for payload in statusPayloads:
status = requests.get(self.uploadURL, params=payload[1])
response = str(list(status.json())[0])
statusList.append([payload[0], response])
return statusList
# Checks for a specific response for each replay in the current batch
# This response is usually a SUCCESS code but can also a fail or pending
def checkForBatchResponse(self, uploadResponses, waitCode):
for i in range(self.batchAmount):
if uploadResponses[i][1] != waitCode:
return False
return True
# Delays the program a user-specified amount of time after the batch is uploaded
# This delay is used to prevent overwhelming the server as well as give other users a chance to have their replays uploaded
def delayAfterUpload(self, delay):
self.statusUpdater.update(
self.statusUpdater.UploadState.queueOverloadDelay, '')
time.sleep(delay)
# Checks if a user-specified amount of time has passed since a time value was stored
# This is primarily used by the status monitor to indicate that the upload is taking too long
def checkForTimeout(self, oldEpoch, length):
return time.time() - oldEpoch >= length
# Gets the unique identifier of a replay file for use by the checkForDuplicateUpload function
# If the user has a pre-created database, the unique identifier is retrieved from it
# If no database is being used, the unique identifer is retrieved using the boxcars parser
def getGUID(self, replay, database=None):
if database is None: database = self.replayDatabase
row = self.databaseManager.getRow(value=replay[1], database=database)
if row is None:
try:
return ReplayDatabaseManager([None] * 9).getReplayGUID(
replay[0], 'boxcars')
except:
return None
else:
return str(row.iloc[0]['GUID'])
# Creates a database if the user specified to do so in an argument on the initial program run
# The database is created in csv format with the row format of:
# Replay name, Replay GUID, Full replay path
def createDatabase(self):
if self.attributes[9] == True:
self.statusUpdater.clearScreen()
ReplayDatabaseManager(self.attributes).exportReplayDatabase()
print('\n\nThe replay database has been created!')
sys.exit()
else:
self.replayDatabase = self.attributes[8]
self.databaseManager.database = self.replayDatabase
