class UploadMongoData():
def __init__(self):
self.settings = Settings()
self.file = FileIOMiddleware()
self.doraemon = Doraemon()
self.log_path = self.settings.LOG_PATH
self.doraemon.createFilePath(self.log_path)
def startUpload(self):
fromFile = self.settings.LOCAL_MONGO_DATA_PATH
toFile = self.settings.REMOTE_MONGO_DATA_PATH
if not os.path.exists(fromFile):
print 'no mongo data file to upload'
return
while os.path.exists(fromFile):
try:
if self.doraemon.sshUpload(
self.settings.IP_WEBSERVER0,
self.settings.PORT_WEBSERVER0,
self.settings.USER_ROOT_WEBSERVER0,
self.settings.USER_ROOT_PASSWORD_WEBSERVER0, fromFile,
toFile):
self.doraemon.deleteFile(fromFile)
message1 = 'Success to upload mongo data file: {0}'.format(
fromFile)
print message1
self.file.logger(self.log_path, message1)
except Exception as e:
message2 = 'Exception {0} to upload mongo data file: {1}'.format(
e.message, fromFile)
print message2
self.file.logger(self.log_path, message2)
class ProcessTimeoutHandler():
def __init__(self):
self.doraemon = Doraemon()
self.file = FileIOMiddleware()
self.settings = Settings()
self.cache_file = self.settings.TIMEOUT_CACHE_FILE
self.timeout = self.settings.PROCESS_TIMEOUT
self.timeout_content = self.settings.PROCESS_TIMEOUT_CONTENT
def updateTimeoutCacheFile(self, processes):
if self.doraemon.isFileExists(self.cache_file):
self.doraemon.deleteFile(self.cache_file)
for process in processes:
tmp = '{0}-{1}-{2}'.format(process.pid, process.pname,
process.past)
self.file.writeToTxtAdd(self.cache_file, tmp)
def getTimeoutCache(self):
result = []
if self.doraemon.isFileExists(self.cache_file):
data = self.file.readFromTxt(self.cache_file)
pidTimeoutList = data.split('\n')
for item in pidTimeoutList:
if self.doraemon.isEmpty(item) is False:
tmp = item.split('-')
if len(tmp) == 3:
result.append(
ProcessTimeoutDto(int(tmp[0]), tmp[1],
float(tmp[2]), False))
return result
def findTarget(self, pid, pids):
result = None
for p in pids:
if p.pid == pid:
result = p
return result
def filterTimeoutProcesses(self, curpids, prepids):
result = []
if self.doraemon.isEmpty(curpids):
return result
for p in curpids:
pre = self.findTarget(p.pid, prepids)
if pre is not None:
if (pre.pname == 'chrome' and self.doraemon.isExceedTimeoutInterval(self.timeout, pre.past)) or \
(pre.pname == 'python' and self.doraemon.isExceedTimeoutInterval(self.timeout_content, pre.past)):
result.append(
ProcessTimeoutDto(pre.pid, pre.pname, pre.past, True))
else:
result.append(p)
return result
def getCurrentProcesses(self):
result = []
pids = psutil.pids()
if len(pids):
print 'No Chrome process.'
for pid in pids:
try:
p = psutil.Process(pid)
pname = p.name()
if pname == 'chrome' or pname == 'python':
print 'Start to store process {0}.'.format(pid)
result.append(
ProcessTimeoutDto(pid, pname, p._create_time, False))
except Exception as e:
print 'Exception {0} to find process: pid - {1}'.format(
e, p.pid)
return result
def processTimeoutProcesses(self, curpids, prepids):
updatedPids = self.filterTimeoutProcesses(curpids, prepids)
notTimeoutProcesses = []
for p in updatedPids:
if p.isTimeout:
try:
print 'kill timeout process: pid - {0}'.format(p.pid)
os.kill(p.pid, signal.SIGKILL)
except Exception as e:
print 'Exception {0} to kill process: pid - {1}'.format(
e, p.pid)
else:
notTimeoutProcesses.append(p)
self.updateTimeoutCacheFile(notTimeoutProcesses)
def start(self):
print 'Start to process.'
self.processTimeoutProcesses(self.getCurrentProcesses(),
self.getTimeoutCache())
class SSHUpload():
def __init__(self):
self.settings = Settings()
self.file = FileIOMiddleware()
self.doraemon = Doraemon()
def writeBack(self, fromFiles):
print "Start to update retry file: {0}".format(
self.settings.RETRY_FILE)
writeBackContent = ''
for file in fromFiles:
writeBackContent = '{0}{1}\n'.format(writeBackContent, file)
self.file.writeToTxtAdd(self.settings.RETRY_FILE, writeBackContent)
print "Finished to update retry file: {0}".format(
self.settings.RETRY_FILE)
def updateRemoveFile(self, fromFiles):
if self.doraemon.isEmpty(fromFiles):
print "No need to update to remove retry file."
return
content = self.readFile()
print "Start to delete retry file: {0}".format(
self.settings.RETRY_FILE)
os.remove(self.settings.RETRY_FILE)
print "Finished to delete retry file: {0}".format(
self.settings.RETRY_FILE)
for file in fromFiles:
if file in content:
del content[content.index(file)]
else:
content.append(file)
self.writeBack(content)
def updateAddFile(self, fromFiles):
if self.doraemon.isEmpty(fromFiles):
print "No need to update to add retry file"
return
content = self.readFile()
if self.doraemon.isEmpty(content):
self.writeBack(fromFiles)
return
print "Start to delete retry file: {0}".format(
self.settings.RETRY_FILE)
os.remove(self.settings.RETRY_FILE)
print "Finished to delete retry file: {0}".format(
self.settings.RETRY_FILE)
for file in content:
if file not in fromFiles:
fromFiles.append(file)
self.writeBack(fromFiles)
def readFile(self):
files = []
isRetryFileExists = os.path.exists(self.settings.RETRY_FILE)
if isRetryFileExists == False:
return files
content = self.file.readFromTxt(self.settings.RETRY_FILE)
if self.doraemon.isEmpty(content):
return files
items = content.split('\n')
for item in items:
if self.doraemon.isEmpty(item):
continue
files.append(item)
return files
def retry(self):
while True:
files = self.readFile()
updateFiles = []
try:
for fromFile in files:
fileParts = re.split(r'[/]', fromFile)
fileName = fileParts[len(fileParts) - 1]
toFile = '{0}/{1}'.format(self.settings.HTML_WEBSERVER0,
fileName)
if self.doraemon.sshUpload(
self.settings.IP_WEBSERVER0,
self.settings.PORT_WEBSERVER0,
self.settings.USER_ROOT_WEBSERVER0,
self.settings.USER_ROOT_PASSWORD_WEBSERVER0,
fromFile, toFile):
updateFiles.append(fromFile)
print 'Success to retry to upload: {0}'.format(
fromFile)
self.updateRemoveFile(updateFiles)
except Exception as e:
self.updateRemoveFile(updateFiles)
print 'Exception {0} to retry to upload: {1}'.format(
e.message, fromFile)
def startUpload(self):
fromFile = '{0}.tar.gz'.format(self.settings.LOCAL_HTML_PATH)
if not os.listdir(self.settings.LOCAL_HTML_PATH) and os.path.exists(
fromFile) is False:
print 'no html file to tar'
return
uploadedList = []
if os.path.exists(fromFile) is False:
uploadedList = self.doraemon.tarList(self.settings.LOCAL_HTML_PATH)
while os.path.exists(fromFile):
try:
if self.doraemon.uploadFileApi(self.settings.UPLOAD_HTML_API,
'local.tar.gz', fromFile):
os.remove(fromFile)
for file in uploadedList:
self.doraemon.deleteFile(file)
print 'Success to upload html file: {0}'.format(fromFile)
except Exception as e:
print 'Exception {0} to upload html file: {1}'.format(
e.message, fromFile)