def getNextPackageToBuild():
with Scheduler.lock:
if Scheduler.stopScheduling:
return None
if not Scheduler.listOfPackagesToBuild:
if Scheduler.event is not None:
Scheduler.event.set()
if Scheduler.listOfPackagesNextToBuild.empty():
Scheduler._getListNextPackagesReadyToBuild()
if Scheduler.listOfPackagesNextToBuild.empty():
return None
packageTup = Scheduler.listOfPackagesNextToBuild.get()
package = packageTup[1]
if not constants.startSchedulerServer and Scheduler.listOfPackagesNextToBuild.qsize(
) > 0:
ThreadPool.activateWorkerThreads(
Scheduler.listOfPackagesNextToBuild.qsize())
Scheduler.listOfPackagesCurrentlyBuilding.add(package)
Scheduler.listOfPackagesToBuild.remove(package)
return package
def getNextPackageToBuild():
Scheduler.logger.info("Waiting to acquire scheduler lock")
with Scheduler.lock:
if Scheduler.stopScheduling:
return None
if not Scheduler.listOfPackagesToBuild:
if Scheduler.event is not None:
Scheduler.event.set()
if Scheduler.listOfPackagesNextToBuild.empty():
Scheduler._getListNextPackagesReadyToBuild()
if Scheduler.listOfPackagesNextToBuild.empty():
return None
packageTup = Scheduler.listOfPackagesNextToBuild.get()
package = packageTup[1]
Scheduler.logger.info("PackagesNextToBuild " + str(packageTup))
if Scheduler.listOfPackagesNextToBuild.qsize() > 0:
ThreadPool.activateWorkerThreads(
Scheduler.listOfPackagesNextToBuild.qsize())
Scheduler.listOfPackagesCurrentlyBuilding.add(package)
Scheduler.listOfPackagesToBuild.remove(package)
return package
def initializeThreadPool(self, statusEvent):
ThreadPool.clear()
ThreadPool.mapPackageToCycle = self.mapPackageToCycle
ThreadPool.listAvailableCyclicPackages = self.listAvailableCyclicPackages
ThreadPool.logger = self.logger
ThreadPool.statusEvent = statusEvent
ThreadPool.pkgBuildType = self.pkgBuildType
def test_server_list(cls, target_list, shared_settings):
"""
Tests connectivity with each server of the target_list and returns
the list of online servers.
"""
# Use a thread pool to connect to each server
thread_pool = ThreadPool()
for target_str in target_list:
thread_pool.add_job((cls._test_server, (target_str, shared_settings)))
nb_threads = min(len(target_list), cls.MAX_THREADS)
thread_pool.start(nb_threads)
# Return valid targets
for (job, target) in thread_pool.get_result():
yield target
# Use None as a sentinel
yield None
# Return invalid targets
for (job, exception) in thread_pool.get_error():
yield exception
thread_pool.join()
return
def test_connectivity(self, timeout):
"""
Tests connectivity with each server of the target_list and returns
the list of online servers.
"""
# Use a thread pool to connect to each server
thread_pool = ThreadPool()
for target_str in self._target_list:
thread_pool.add_job((self._test_server, (target_str, timeout)))
nb_threads = min(len(self._target_list), self.MAX_THREADS)
thread_pool.start(nb_threads)
# Recover valid targets
for (job, target) in thread_pool.get_result():
self._targets_OK.append(target)
yield target
# Store invvalid targets
for (job, exception) in thread_pool.get_error():
self._targets_ERR.append(exception)
thread_pool.join()
return
def main():
appName = sys.argv[0]
logging.basicConfig(
# filename = './log/' + appName + '_' + time.strftime("%Y%m%d_%H%M%S") + '.log',
datefmt = '%Y-%m%d %H:%M:%S',
format = '%(asctime)s | %(levelname)s | %(name)s | %(message)s',
level = logging.INFO
)
func_name = sys._getframe().f_code.co_name
logger = logging.getLogger(func_name)
logger.info('Started')
parseArgs()
logger.debug('requests.session')
session = requests.session()
# `mount` a custom adapter that retries failed connections for HTTP and HTTPS requests.
session.mount("http://", requests.adapters.HTTPAdapter(max_retries=10))
logger.debug('session.post')
r = session.post(LOGIN_URL, data = LOGIN_PARAM)
idResultEnd = idResult
idResultBegin = idResult - quantResult
# 1) Init a Thread pool with the desired number of threads
logger.debug('ThreadPool')
pool = ThreadPool(10)
logger.debug('for idAtual in xrange(%d, %d, -1)' % (idResultEnd, idResultBegin))
for idAtual in xrange(idResultEnd, idResultBegin, -1):
# 2) Add the task to the queue
pool.add_task(downloadResult, session, idAtual)
# 3) Wait for completion
pool.wait_completion()
###
logger.info('Finished')
def initializeThreadPool(self,statusEvent):
ThreadPool.clear()
ThreadPool.mapPackageToCycle=self.mapPackageToCycle
ThreadPool.listAvailableCyclicPackages=self.listAvailableCyclicPackages
ThreadPool.listBuildOptionPackages=self.listBuildOptionPackages
ThreadPool.pkgBuildOptionFile=self.pkgBuildOptionFile
ThreadPool.logger=self.logger
ThreadPool.statusEvent=statusEvent
def __init__(self, is_debug=False):
self.prev_day = {}
self.thread_pool = ThreadPool(
is_debug,
int(cfg.read('max_threads')),
int(cfg.read('max_buf')),
)
# 注意先清空先前留下的下载任务
# TODO: 改为继续执行先前未完成的任务
db.Execute("DELETE FROM `CurrentTask`")
def test_connectivity(self, timeout):
"""
Tests connectivity with each server of the target_list and returns
the list of online servers.
"""
# Use a thread pool to connect to each server
thread_pool = ThreadPool()
for target_str in self._target_list:
thread_pool.add_job((self._test_server,
(target_str, timeout)))
nb_threads = min(len(self._target_list), self.MAX_THREADS)
thread_pool.start(nb_threads)
# Recover valid targets
for (job, target) in thread_pool.get_result():
self._targets_OK.append(target)
yield target
# Store invvalid targets
for (job, exception) in thread_pool.get_error():
self._targets_ERR.append(exception)
thread_pool.join()
return
def buildGivenPackages(self, listPackages):
returnVal = self.calculateParams(listPackages)
if not returnVal:
self.logger.error(
"Unable to set paramaters. Terminating the package manager.")
return False
statusEvent = threading.Event()
numWorkerThreads = self.calculatePossibleNumWorkerThreads()
if numWorkerThreads > 8:
numWorkerThreads = 8
if numWorkerThreads == 0:
return False
self.initializeScheduler(statusEvent)
self.initializeThreadPool(statusEvent)
i = 0
while i < numWorkerThreads:
workerName = "WorkerThread" + str(i)
ThreadPool.addWorkerThread(workerName)
ThreadPool.startWorkerThread(workerName)
i = i + 1
statusEvent.wait()
Scheduler.stopScheduling = True
self.logger.info("Waiting for all remaining worker threads")
listWorkerObjs = ThreadPool.getAllWorkerObjects()
for w in listWorkerObjs:
w.join()
setFailFlag = False
allPackagesBuilt = False
if Scheduler.isAnyPackagesFailedToBuild():
setFailFlag = True
if Scheduler.isAllPackagesBuilt():
allPackagesBuilt = True
if setFailFlag:
self.logger.error("Some of the packages failed:")
self.logger.error(Scheduler.listOfFailedPackages)
return False
if not setFailFlag:
if allPackagesBuilt:
self.logger.info("All packages built successfully")
else:
self.logger.error("Build stopped unexpectedly.Unknown error.")
return False
self.logger.info("Terminated")
return True
def __init__(self, device):
"""
Constructor.
@type device: Device
@param device: the device which owns this thread
"""
Thread.__init__(self, name="Device Thread %d" % device.device_id)
self.device = device
self.thread_pool = ThreadPool(8)
def __init__(self):
from ThreadPool import ThreadPool
self.__threadPool = ThreadPool(self.maxThreads, self.maxQueue)
self.__currentJob = None
self.__cache = {}
self.__defaultETA = ''
self.__TIME = 0
self.__ETA = 1
def buildGivenPackages (self, listPackages):
returnVal=self.calculateParams(listPackages)
if not returnVal:
self.logger.error("Unable to set paramaters. Terminating the package manager.")
return False
statusEvent=threading.Event()
numWorkerThreads=self.calculatePossibleNumWorkerThreads()
if numWorkerThreads > 8:
numWorkerThreads = 8
if numWorkerThreads == 0:
return False
self.initializeScheduler(statusEvent)
self.initializeThreadPool(statusEvent)
i=0
while i < numWorkerThreads:
workerName="WorkerThread"+str(i)
ThreadPool.addWorkerThread(workerName)
ThreadPool.startWorkerThread(workerName)
i = i + 1
statusEvent.wait()
Scheduler.stopScheduling=True
self.logger.info("Waiting for all remaining worker threads")
listWorkerObjs=ThreadPool.getAllWorkerObjects()
for w in listWorkerObjs:
w.join()
setFailFlag=False
allPackagesBuilt=False
if Scheduler.isAnyPackagesFailedToBuild():
setFailFlag=True
if Scheduler.isAllPackagesBuilt():
allPackagesBuilt=True
if setFailFlag:
self.logger.error("Some of the packages failed:")
self.logger.error(Scheduler.listOfFailedPackages)
return False
if not setFailFlag:
if allPackagesBuilt:
self.logger.info("All packages built successfully")
else:
self.logger.error("Build stopped unexpectedly.Unknown error.")
return False
self.logger.info("Terminated")
return True
def buildGivenPackages(self, listPackages, buildThreads):
if constants.rpmCheck:
alreadyBuiltRPMS = self.readAlreadyAvailablePackages()
listPackages = list(
set(listPackages)
| (set(constants.listMakeCheckRPMPkgtoInstall) -
set(alreadyBuiltRPMS)))
returnVal = self.calculateParams(listPackages)
if not returnVal:
self.logger.error(
"Unable to set paramaters. Terminating the package manager.")
raise Exception("Unable to set paramaters")
statusEvent = threading.Event()
self.initializeScheduler(statusEvent)
self.initializeThreadPool(statusEvent)
i = 0
while i < buildThreads:
workerName = "WorkerThread" + str(i)
ThreadPool.addWorkerThread(workerName)
ThreadPool.startWorkerThread(workerName)
i = i + 1
statusEvent.wait()
Scheduler.stopScheduling = True
self.logger.info("Waiting for all remaining worker threads")
listWorkerObjs = ThreadPool.getAllWorkerObjects()
for w in listWorkerObjs:
w.join()
setFailFlag = False
allPackagesBuilt = False
if Scheduler.isAnyPackagesFailedToBuild():
setFailFlag = True
if Scheduler.isAllPackagesBuilt():
allPackagesBuilt = True
if setFailFlag:
self.logger.error("Some of the packages failed:")
self.logger.error(Scheduler.listOfFailedPackages)
raise Exception("Failed during building package")
if not setFailFlag:
if allPackagesBuilt:
self.logger.info("All packages built successfully")
else:
self.logger.error("Build stopped unexpectedly.Unknown error.")
raise Exception("Unknown error")
self.logger.info("Terminated")
def test():
print('start testing')
tp = ThreadPool()
for i in range(15):
time.sleep(0.1)
tp.add_job(test_job, i)
t = time.time()
tp.wait_for_complete()
print('s:'+str(time.time()-t))
print('end testing')
def compare_site_thread_csv(file, progress_var=None, step=100.0):
if status["INTERFACE_MODE"]:
thread_pool_csv = ThreadPool(settings["THREADPOOL_SIZE"])
else:
thread_pool_csv = ThreadPool(20)
f = open(file, 'r')
# calculate the step for each site
row_count = sum(1 for row in f)
site_step = step / row_count
f.close()
f = open(file, 'r')
rows = csv.reader(f)
for row in rows:
compare_site_thread(row[0],
row[1],
progress_var=progress_var,
step=site_step,
thread_pool_csv=thread_pool_csv)
# check program status
if status["INTERFACE_MODE"] and not status["CHECKING_STATUS"]:
f.close()
return
f.close()
thread_pool_csv.wait_completion()
thread_pool_csv.destroy()
def handle_init_snapshot(cls, incoming_message):
# Temporarily pause the money transfer
MoneyTransferThread.set_enabled(False)
# Grab the lock, since the current_snapshots data structure we modify ahead is shared between threads
cls.marker_handler_lock.acquire()
# Make sure that the new snapshot_id doesn't already exist
snapshot_id = incoming_message.init_snapshot.snapshot_id
if snapshot_id in cls.current_snapshots:
print "ERROR! Got init snapshot again for snapshot id : " + str(
snapshot_id)
return
print "Got init_snapshot msg (snapshot_id: " + str(snapshot_id) + ")"
# Record local state
current_balance = BankVault.get_balance()
state = {"local": current_balance}
cls.current_snapshots[snapshot_id] = state
# Create marker message
marker_msg = bank_pb2.Marker()
marker_msg.snapshot_id = snapshot_id
pb_msg = bank_pb2.BranchMessage()
pb_msg.marker.CopyFrom(marker_msg)
# Send marker message to everyone else
total_peers = ThreadPool.get_thread_count()
for i in range(total_peers):
a_friend = ThreadPool.get_thread(i)
if not a_friend:
continue
print "Sending marker message to : " + str(a_friend.remote_branch_name) + " (snapshot_id: "\
+ str(snapshot_id) + ")"
a_friend.send_msg_to_remote(pb_msg)
# Start recording all incoming activity
a_friend.add_recorder(snapshot_id)
# Release the lock, we're done modifying the shared data structure
cls.marker_handler_lock.release()
# Resume the money transfer thread
MoneyTransferThread.set_enabled(True)
def buildGivenPackages (self, listPackages, buildThreads):
if constants.rpmCheck:
alreadyBuiltRPMS=self.readAlreadyAvailablePackages()
listPackages=list(set(listPackages)|(set(constants.listMakeCheckRPMPkgtoInstall)-set(alreadyBuiltRPMS)))
returnVal=self.calculateParams(listPackages)
if not returnVal:
self.logger.error("Unable to set paramaters. Terminating the package manager.")
raise Exception("Unable to set paramaters")
statusEvent=threading.Event()
self.initializeScheduler(statusEvent)
self.initializeThreadPool(statusEvent)
i=0
while i < buildThreads:
workerName="WorkerThread"+str(i)
ThreadPool.addWorkerThread(workerName)
ThreadPool.startWorkerThread(workerName)
i = i + 1
statusEvent.wait()
Scheduler.stopScheduling=True
self.logger.info("Waiting for all remaining worker threads")
listWorkerObjs=ThreadPool.getAllWorkerObjects()
for w in listWorkerObjs:
w.join()
setFailFlag=False
allPackagesBuilt=False
if Scheduler.isAnyPackagesFailedToBuild():
setFailFlag=True
if Scheduler.isAllPackagesBuilt():
allPackagesBuilt=True
if setFailFlag:
self.logger.error("Some of the packages failed:")
self.logger.error(Scheduler.listOfFailedPackages)
raise Exception("Failed during building package")
if not setFailFlag:
if allPackagesBuilt:
self.logger.info("All packages built successfully")
else:
self.logger.error("Build stopped unexpectedly.Unknown error.")
raise Exception("Unknown error")
self.logger.info("Terminated")
def __init__(self, workDir, setup, esPath, token, experiment, userid, sitename, outputDir=None, yodaToOS=False, threads=10, isDaemon=False, process=0, totalProcess=1):
self.__workDir = workDir
self.__updateEventRangesDir = os.path.join(self.__workDir, 'updateEventRanges_%s' % process)
if not os.path.exists(self.__updateEventRangesDir):
os.makedirs(self.__updateEventRangesDir)
self.__logFile = os.path.join(workDir, 'EventStager.log')
self.__setup = setup
self.__siteMover = S3ObjectstoreSiteMover(setup, useTimerCommand=False)
self.__esPath = esPath
self.__token = token
self.__experiment = experiment
self.__outputDir = outputDir
self.__userid = userid
self.__sitename = sitename
self.__report = getInitialTracingReport(userid=self.__userid, sitename=self.__sitename, dsname=None, eventType="objectstore", analysisJob=False, jobId=None, jobDefId=None, dn=self.__userid)
self.__num_stagingFile = 0
self.__eventRanges = {}
self.__eventRanges_staged = {}
self.__eventRanges_faileStaged = {}
self.__eventStager = None
self.__canFinish = False
self.__status = 'new'
self.__threads = threads
self.__isDaemon = isDaemon
self.__startTime = time.time()
self.__processedJobs = []
self.__handlingOthers = 0
self.__otherProcesses = []
self.__startWait = None
self.__waitTime = 15 * 60 # 15 minutes
self.__yodaToOS = yodaToOS
if not os.environ.has_key('PilotHomeDir'):
os.environ['PilotHomeDir'] = os.path.dirname(__file__)
self.__process = process
self.__totalProcess = totalProcess
self.__siteMover.setup(experiment)
self.__threadpool = ThreadPool(self.__threads)
logging.info("Init EventStager workDir %s setup %s esPath %s token %s experiment %s userid %s sitename %s threads %s outputDir %s isDaemond %s" % (self.__workDir, self.__setup, self.__esPath, self.__token, self.__experiment, self.__userid, self.__sitename, self.__threads, self.__outputDir, self.__isDaemon))
def main():
config = Config.config
##线程池
pool = ThreadPool(config['threatNum'])
##会话
session = Session(config['session'])
##爬虫开始
spider = Spider(config,session)
##获取首页索引页 进队列 多线程并发下载
lists = spider.getIndex()
if lists:
for li in lists:
pool.add2queue(spider.getImgs,li)
pool.q.join()
else:
print('抓取异常，没有数据')
def setup(self, job):
try:
self.__jobId = job.get("JobId", None)
self.__yodaToOS = job.get('yodaToOS', False)
self.__yodaToZip = job.get('yodaToZip', False)
self.__zipFileName = job.get('zipFileName', None)
self.__zipEventRangesName = job.get('zipEventRangesName', None)
self.__tmpLog.debug("Rank %s: zip file %s" % (self.__rank, self.__zipFileName))
self.__tmpLog.debug("Rank %s: zip event range file %s" % (self.__rank, self.__zipEventRangesName))
if self.__zipFileName is None or self.__zipEventRangesName is None:
self.__tmpLog.debug("Rank %s: either zipFileName(%s) is None or zipEventRanagesName(%s) is None, will not use zip output" % (self.__rank, self.__zipFileName, self.__zipEventRangesName))
self.__yodaToZip = False
self.__copyOutputToGlobal = job.get('copyOutputToGlobal', False)
if self.__yodaToOS:
setup = job.get('setup', None)
self.__esPath = job.get('esPath', None)
self.__os_bucket_id = job.get('os_bucket_id', None)
self.__report = getInitialTracingReport(userid='Yoda', sitename='Yoda', dsname=None, eventType="objectstore", analysisJob=False, jobId=None, jobDefId=None, dn='Yoda')
self.__siteMover = objectstoreSiteMover(setup, useTimerCommand=False)
self.__cores = int(job.get('ATHENA_PROC_NUMBER', 1))
try:
self.__stageout_threads = int(job.get('stageout_threads', None))
except:
self.__stageout_threads = self.__cores/8
self.__tmpLog.debug("Rank %s: start threadpool with %s threads" % (self.__rank, self.__stageout_threads))
self.__threadpool = ThreadPool(self.__stageout_threads)
except:
self.__tmpLog.error("Failed to setup Droid stager: %s" % str(traceback.format_exc()))
def __init__(self):
from ThreadPool import ThreadPool
self.__threadPool = ThreadPool(self.maxThreads, self.maxQueue)
self.__currentJob = None
self.__cache = {}
self.__queue = {}
self.__defaultLine = ""
self.__defaultLLU = ""
# Position of data in self.__cache
self.__TIME = 0
self.__PATH = 1
self.__LINE = 2
self.__LLU = 3
def getWantToSeeMovies(self):
searchURL = self.baseURL + '/usuario/' + self.username + '/quero-ver/'
moviesVec = []
netflixVec = []
nThreads = 8
threadPool = ThreadPool(nThreads)
threadPool.startWorking()
def parsePage(pageUrl):
wantToSeeCatalogueHTML = urllib.request.urlopen(
urllib.request.Request(pageUrl, headers=self.hdr))
catalogueSoup = BeautifulSoup(wantToSeeCatalogueHTML,
'html.parser')
print(pageUrl)
#looping through each movie in the current page
for movieDiv in catalogueSoup.findAll(
'li', {'class': 'span2 movie_list_item'}):
divSoup = BeautifulSoup(str(movieDiv), 'html.parser')
moviehref = str(divSoup.find("a")['href'])
print(moviehref)
movieURL = self.baseURL + moviehref
threadPool.putInQueue(parseMovie, {"movieURL": movieURL})
def parseMovie(movieURL):
movie = {}
moviePageHtml = urllib.request.urlopen(
urllib.request.Request(movieURL, headers=self.hdr))
moviePageSoup = BeautifulSoup(moviePageHtml, 'html.parser')
movie['name'] = str(
moviePageSoup.find('h2', {
'class': 'movie-original-title'
}).string)
movie['duration'] = str(
moviePageSoup.find('span', {
'class': 'running_time'
}).string)
print(movie)
moviesVec.append(movie)
threadPool.putInQueue(checkNetflix, {'title': movie['name']})
def checkNetflix(title):
netflixWrapper = NetflixWrapper()
resp = netflixWrapper.isTitleInNetflix(title)
if (resp[0]):
netflixVec.append(resp[1])
for i in range(1, self.getWantToSeePages() + 1):
pageUrl = searchURL + '?pagina=' + str(i)
threadPool.putInQueue(parsePage, {'pageUrl': pageUrl})
#block until all tasks are done
threadPool.end()
return [moviesVec, netflixVec]
class Spider(object):
unvisited_urls = dict() # 存储未访问列表
visited_urls = dict()
task_queue = Queue() # 任务队列
def __init__(self, seed):
self.unvisited_urls = seed
self.gather = Gather()
self.url = ''
self.pool = ThreadPool(self.task_queue, 2) # 启动4个线程
self.lock = threading.Lock() # 初始化锁
'''
start 从未访问url列表中取出一条未访问过的url，与run一起打包放到任务队列中
'''
def start(self):
# 在这里读取配置文件
for key in self.unvisited_urls:
self.task_queue.put((self.run, key))
self.pool.wait_all_complete()
# 在这里把已完成队列和未完成队列写入文件中。
def run(self, url, *, save_dir, name): # 工作线程，下载网页，保存网页，提取新的链接,线程需要提供编号，以便存储(命名参数)
# time.sleep(5) # 防止ip被封
result = self.gather.download_webpage(url) # 下载网页
if forbidden_count >= 50: # 结束整个程序
print("exit function")
return False
if result[0] and self.gather.doc_save(result[1], save_dir, name): # 保存网页
self.visited_urls[url] = True # 标记为已经访问过的网址
new_url = self.gather.extra_urls(result[1]) # 提取新的url
self.lock.acquire()
for u in new_url: # 如果该链接既没有访问过，也不存在于待访问列表中，添加网址,这里在多线程中不安全
if not self.visited_urls.get(u) and not self.unvisited_urls.get(u):
# print("%s add" % u)
self.unvisited_urls[u] = True
self.task_queue.put((self.run, u))
print("任务队列大小：%d" % self.task_queue.qsize())
print("已完成任务量：%d" % len(self.visited_urls))
self.lock.release()
self.lock.acquire()
self.unvisited_urls.pop(url) # 在未访问列表中删除该网址
self.lock.release() # 释放
print()
return True
def run(self):
global init_received
if not init_received:
# First message we'll receive will always be init
init_received = True
init_message = bank_pb2.BranchMessage()
# Accept the init message from controller
init_message_from_socket = self.client_socket.recv(
MAX_REQUEST_SIZE)
if len(init_message_from_socket) == 0:
# The controller just opened the connection but didn't send any message
print(
"ERROR! The controller didn't send anything as Init Message!"
)
self.client_socket.close()
return 0
init_message.ParseFromString(init_message_from_socket)
# Connect to the list of branches got from init_branch message
connect_to_branches(init_message.init_branch)
# Start the thread to send money to these branches
mt_thread = MoneyTransferThread(init_message.init_branch.balance)
mt_thread.daemon = True
mt_thread.start()
# Start handling further incoming messages from controller
self.start_handling_messages()
# Else, we're connected to a branch
# Let's get introduced.
self.client_socket.send(branch_name)
remote_branch_name = self.client_socket.recv(MAX_REQUEST_SIZE)
self.remote_branch_name = remote_branch_name
print "Connected to : " + str(remote_branch_name)
# Add this thread object to the pool of threads such that it can be accessed whenever necessary
ThreadPool.add_thread(self)
# Start handling further incoming messages from connected branch
self.start_handling_messages()
def test_initialization(self):
threadpool_list = []
total_cnt_expected = 0
for i in range(0, 5):
total_cnt_expected += i
threadpool_list.append(ThreadPool(i, 3, extra_debug_info=True))
cnt = self.get_worker_thread_cnt()
self.assertEqual(cnt, total_cnt_expected)
def __init__(self,
workDir,
setup,
esPath,
token,
experiment,
userid,
sitename,
outputDir=None,
yodaToOS=False,
threads=10,
isDaemon=False):
super(MVEventStager,
self).__init__(workDir, setup, esPath, token, experiment, userid,
sitename, outputDir, yodaToOS, threads, isDaemon)
self.__workDir = workDir
self.__updateEventRangesDir = os.path.join(self.__workDir,
'MVupdateEventRanges')
if not os.path.exists(self.__updateEventRangesDir):
os.makedirs(self.__updateEventRangesDir)
self.__logFile = os.path.join(workDir, 'EventStager.log')
self.__setup = setup
self.__siteMover = objectstoreSiteMover(setup)
self.__esPath = esPath
self.__token = token
self.__experiment = experiment
self.__outputDir = outputDir
self.__userid = userid
self.__sitename = sitename
self.__report = getInitialTracingReport(userid=self.__userid,
sitename=self.__sitename,
dsname=None,
eventType="objectstore",
analysisJob=False,
jobId=None,
jobDefId=None,
dn=self.__userid)
self.__eventRanges = {}
self.__eventRanges_staged = {}
self.__eventRanges_faileStaged = {}
self.__eventStager = None
self.__canFinish = False
self.__status = 'new'
self.__threads = threads
self.__yodaToOS = yodaToOS
self.__isDaemon = isDaemon
self.__threadpool = ThreadPool(self.__threads)
logging.info(
"Init EventStager workDir %s setup %s esPath %s token %s experiment %s userid %s sitename %s threads %s outputDir %s isDaemond %s"
% (self.__workDir, self.__setup, self.__esPath, self.__token,
self.__experiment, self.__userid, self.__sitename,
self.__threads, self.__outputDir, self.__isDaemon))
def GetMatchPage(season, league, matches):
'''
获取指定联赛的Match页面
'''
tp = ThreadPool()
print('Start updating ' + season + ' ' + league + ' matches...')
dirLeague = Global.Dir_Root + season + '\\' + league
for matchID in sorted(matches.keys()):
urlMatch = matches.get(matchID)
tp.add_job(GetPageText, dirLeague, matchID, urlMatch, len(matches))
tp.wait_for_complete()
if len(matches) > 0:
print('\n' + season + ' ' +league + ' updating complete!\n')
else:
print(season + ' ' + league + ' updating complete!\n')
def getBackLinks(self, workDir, linksGraph):
tp = ThreadPool(3)
linksIndexes = self.__getStringIndexDict(linksGraph)
self.__mapStage(linksGraph, linksIndexes, tp, workDir)
self.__reduceStage(tp, workDir)
backLinksIndexesGraph = self.__getBackLinksFromDirStruct(workDir)
backLinksGraph = self.__getBackLinksGraph(backLinksIndexesGraph,
linksIndexes)
return backLinksGraph
def test_task_execution(self):
global cnt_handle
thread_pool = ThreadPool()
task_list = []
total_cnt_expected = 10
for i in range(total_cnt_expected):
task_list.append(TaskTest("task %s" % i, self.task_handle))
for task in task_list:
thread_pool.add_task(task)
time_start = time.time()
while (True):
if time.time() - time_start > 1.0:
break
if cnt_handle == total_cnt_expected:
break
self.assertEqual(cnt_handle, total_cnt_expected)
def OnExecute(self, button_desc):
if "run_detect_button" == button_desc:
desc_fail = "运行检查程序返回失败\r\n"
desc_success = "所有设备运行检查程序返回成功\r\n"
cmds_txt = "./cmds/cmds_detect.txt"
if "scan_result_button" == button_desc:
desc_fail = "查看检查结果返回失败\r\n"
desc_processexit = "设备检测程序异常退出\r\n"
desc_success = "所有设备查看检查结果返回成功\r\n"
cmds_txt = "./cmds/cmds_scan.txt"
ips = self.ParseIp(self.ip.GetValue().encode('utf-8').split(';'))
if False == ips:
return
cmds = getContent(cmds_txt,"lines")
if False == cmds:
return
pool = ThreadPool( _THREAD_POOL_SIZE )
for ip in ips:
pool.addTask( telnetTask, ip = ip, cmds = cmds)
#必须放在pool.addFinishCb( finish_event,pool=pool)之前
def finish_event( *args, **kwargs ):
pool = kwargs['pool']
result = pool.show()
# [[ip,[ret1,ret2]],...]
result_to_show = ""
for ip_retlist in result:
ip = ip_retlist[0]
retlist = ip_retlist[1]
if ["fail"] == retlist:
result_to_show += ip + " Telnet时抛出了异常\r\n"
else:
t = retlist[len(retlist)-1].split("\r\n")
status = t[len(t)-2]
if "0" == status:
pass
elif "1" == status:
result_to_show += ip + desc_fail#desc_fail能访问到
elif "2" == status:
result_to_show += ip + desc_processexit
else:
result_to_show += ip + " 脚本返回意料之外的值\r\n"
last_cmd_result = retlist[len(retlist)-1]
logging.warning(toUnicode(last_cmd_result))
if "" == result_to_show:
result_to_show = desc_success
logging.warning(result_to_show.decode('utf-8'))
result_to_show = self.result_text.GetValue()+getCurTime().decode("ascii")+" ".decode("utf-8")+result_to_show.decode("utf-8")
self.result_text.SetValue(result_to_show)
self.ButtonStatus("Enable")
pool.addFinishCb( finish_event,pool=pool)
pool.start()
self.ButtonStatus("Disable")
def test_thread_pool():
pool = ThreadPool(2, 0.5)
for i in range(2):
handle = login.login('ssh', 'root', 'n', '10.20.60.23')
task = (handle.execute_cmd, ('uname -a',), {})
pool.add_task(task)
print pool.get_result()
pool.task_join()
class DeviceThread(Thread):
"""
Class that implements the device's worker thread.
"""
def __init__(self, device):
"""
Constructor.
@type device: Device
@param device: the device which owns this thread
"""
Thread.__init__(self, name="Device Thread %d" % device.device_id)
self.device = device
self.thread_pool = ThreadPool(8)
def run(self):
self.thread_pool.set_device(self.device)
while True:
# get the current neighbourhood
neighbours = self.device.supervisor.get_neighbours()
if neighbours is None:
break
# stay in this loop until the timepoint ends
while True:
# if we already have scripts or the timepoint is finished
if self.device.scripts_arrived or self.device.timepoint_done.wait(
):
if self.device.scripts_arrived:
self.device.scripts_arrived = False
# run scripts received until now
for (script, location) in self.device.scripts:
self.thread_pool.submit(neighbours, script,
location)
else:
self.device.timepoint_done.clear()
self.device.scripts_arrived = True
break
# wait for thread pool to finish it's job
self.thread_pool.wait_threads()
# wait for all devices to reach this point
self.device.barrier.wait()
# close all threads in the thread pool
self.thread_pool.end_threads()
class Worker(QThread):
update_ui_signal = pyqtSignal()
def __init__(self, parent=None):
QThread.__init__(self, parent)
self.toDisplay = Queue()
self.threadpool = ThreadPool(max_workers=cpu_count())
def __del__(self):
self.threadpool.shutdown()
def compress_file(self, images, showapp, verbose, imagelist):
"""Start the worker thread."""
for image in images:
#FIXME:http://code.google.com/p/pythonthreadpool/issues/detail?id=5
time.sleep(0.05)
self.threadpool.add_job(image.compress,
None,
return_callback=self.toDisplay.put)
self.showapp = showapp
self.verbose = verbose
self.imagelist = imagelist
self.start()
def run(self):
"""Compress the given file, get data from it and call update_table."""
tp = self.threadpool
while self.showapp or not (tp._ThreadPool__active_worker_count == 0
and tp._ThreadPool__jobs.empty()):
image = self.toDisplay.get()
self.update_ui_signal.emit()
if not self.showapp and self.verbose: # we work via the commandline
if image.retcode == 0:
ir = ImageRow(image)
print("File: " + ir['fullpath'] + ", Old Size: " +
ir['oldfilesizestr'] + ", New Size: " +
ir['newfilesizestr'] + ", Ratio: " + ir['ratiostr'])
else:
print("[error] {} could not be compressed".format(
image.fullpath),
file=sys.stderr)
class Service(object):
def __init__(self):
self.__pool = ThreadPool()
self.__chatModel = ChatModel()
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind((serverLocalHost, serverLocalPort))
self.__socket = s
def start_service(self):
self.__socket.listen(10)
while True:
connection, address = self.__socket.accept()
task = SocketTask(connection, address, self.__chatModel)
self.__pool.execute(task)
def stop_service(self):
self.__pool.shutdown()
self.__chatModel.close_resource()
self.__socket.close()
class Worker(QThread):
update_ui_signal = pyqtSignal()
def __init__(self, parent=None):
QThread.__init__(self, parent)
self.toDisplay = Queue()
self.threadpool = ThreadPool(max_workers=cpu_count())
def __del__(self):
self.threadpool.shutdown()
def compress_file(self, images, showapp, verbose, imagelist):
"""Start the worker thread."""
for image in images:
#FIXME:http://code.google.com/p/pythonthreadpool/issues/detail?id=5
time.sleep(0.05)
self.threadpool.add_job(image.compress, None,
return_callback=self.toDisplay.put)
self.showapp = showapp
self.verbose = verbose
self.imagelist = imagelist
self.start()
def run(self):
"""Compress the given file, get data from it and call update_table."""
tp = self.threadpool
while self.showapp or not (tp._ThreadPool__active_worker_count == 0 and
tp._ThreadPool__jobs.empty()):
image = self.toDisplay.get()
self.update_ui_signal.emit()
if not self.showapp and self.verbose: # we work via the commandline
if image.retcode == 0:
ir = ImageRow(image)
print("File: " + ir['fullpath'] + ", Old Size: "
+ ir['oldfilesizestr'] + ", New Size: "
+ ir['newfilesizestr'] + ", Ratio: " + ir['ratiostr'])
else:
print("[error] {} could not be compressed".format(image.fullpath), file=sys.stderr)
def add_review_record():
global hotel_counter
read_from_mongoDB("hotel_listing")
print("No. of hotel in name list provided:", len(hotel_name_list))
print("No. of hotel url found on TripAdvisor website:", len(hotel_name))
if len(hotel_name) == len(hotel_name_list):
print('Hotel list matched.')
# for i in hotel_url:
# # print(i)
# get_hotel_review(i)
pool = ThreadPool(REVIEW_THREADS)
while hotel_counter < len(hotel_name):
print('Hotel no.:', hotel_counter + 1)
print(hotel_name[hotel_counter])
print(hotel_url[hotel_counter])
try:
param1 = [hotel_url[hotel_counter]]
pool.map(get_hotel_review, param1)
pool.wait_completion()
#get_hotel_review(hotel_url[hotel_counter])
hotel_counter += 1
except:
print("Error: unable to queue for get_hotel_review")
else:
print('Hotel list mismatched.')
def _buildPackages(self, buildThreads):
statusEvent = threading.Event()
self._initializeScheduler(statusEvent)
self._initializeThreadPool(statusEvent)
for i in range(0, buildThreads):
workerName = "WorkerThread" + str(i)
ThreadPool.addWorkerThread(workerName)
ThreadPool.startWorkerThread(workerName)
statusEvent.wait()
Scheduler.stopScheduling = True
self.logger.debug("Waiting for all remaining worker threads")
ThreadPool.join_all()
setFailFlag = False
allPackagesBuilt = False
if Scheduler.isAnyPackagesFailedToBuild():
setFailFlag = True
if Scheduler.isAllPackagesBuilt():
allPackagesBuilt = True
if setFailFlag:
self.logger.error("Some of the packages failed:")
self.logger.error(Scheduler.listOfFailedPackages)
raise Exception("Failed during building package")
if not setFailFlag:
if allPackagesBuilt:
self.logger.debug("All packages built successfully")
else:
self.logger.error("Build stopped unexpectedly.Unknown error.")
raise Exception("Unknown error")
def main(condicion_venta, estado, threads):
start = datetime.now()
try:
condicion_venta = condicion_venta.replace('-', '_')
estado = estado.replace('-', '_')
condicion_venta = CondicionVenta[condicion_venta]
estado = Estado[estado]
max_threads = int(threads)
print('Parámetros de búsqueda')
print('Venta : {0}'.format(condicion_venta.value))
print('Vigente: {0}'.format(estado.value))
except KeyError:
print(
'No fue posible determinar la condicion de venta o estado de medicamentos a procesar'
)
return 1
except ValueError:
print('No se proporcionó un número de hilos de ejecución válido')
return 1
thread = IspParser(sale_terms=condicion_venta, status=estado)
max_pages = thread.pages_count
pool = ThreadPool(max_threads, IspParser)
for i in range(1, max_pages + 1):
pool.add_task({
'sale_terms': condicion_venta,
'status': estado,
'page_number': i
})
pool.wait_completion()
end = datetime.now()
print('Tiempo transcurrido: {0}'.format(end - start))
def get_stock_patch(self):
print('get_stock_patch')
self.raw_data = ''
tp = ThreadPool(10)
path = 'http://hq.sinajs.cn/list='
str = ''
c = 0
for i in self.get_stock_names():
c = c + 1
str += i
str += ','
if c < 200:
continue
tp.add_worker(self.thread_task, path + str, self.thread_task_cb)
str = ''
c = 0
if len(str) > 0:
tp.add_worker(self.thread_task, path + str, self.thread_task_cb)
tp.pool_start()
tp.pool_join()
print('get_stock_patch_done')
return self.raw_data
def getNextPackageToBuild():
with Scheduler.lock:
if Scheduler.stopScheduling:
return None
if not Scheduler.listOfPackagesToBuild:
if Scheduler.event is not None:
Scheduler.event.set()
if Scheduler.listOfPackagesNextToBuild.empty():
Scheduler._getListNextPackagesReadyToBuild()
if Scheduler.listOfPackagesNextToBuild.empty():
return None
packageTup = Scheduler.listOfPackagesNextToBuild.get()
package = packageTup[1]
if Scheduler.listOfPackagesNextToBuild.qsize() > 0:
ThreadPool.activateWorkerThreads(
Scheduler.listOfPackagesNextToBuild.qsize())
Scheduler.listOfPackagesCurrentlyBuilding.add(package)
Scheduler.listOfPackagesToBuild.remove(package)
return package
def get_stock_patch(self):
print('get_stock_patch')
self.raw_data = ''
tp = ThreadPool(10)
path = 'http://hq.sinajs.cn/list='
str = ''
c = 0
for i in self.get_stock_names():
c = c + 1
str += i
str += ','
if c < 200:
continue
tp.add_worker(self.thread_task, path + str, self.thread_task_cb)
str = ''
c = 0
if len(str) > 0:
tp.add_worker(self.thread_task, path + str, self.thread_task_cb)
tp.pool_start()
tp.pool_join()
print('get_stock_patch_done')
return self.raw_data
def getWantToSeeMovies(self):
searchURL = self.baseURL + '/usuario/' + self.username + '/quero-ver/';
moviesVec = []
netflixVec = []
nThreads = 8
threadPool = ThreadPool(nThreads)
threadPool.startWorking()
def parsePage(pageUrl):
wantToSeeCatalogueHTML = urllib.request.urlopen(urllib.request.Request(pageUrl, headers=self.hdr))
catalogueSoup = BeautifulSoup(wantToSeeCatalogueHTML, 'html.parser')
print(pageUrl)
#looping through each movie in the current page
for movieDiv in catalogueSoup.findAll('li', { 'class': 'span2 movie_list_item'}):
divSoup = BeautifulSoup(str(movieDiv), 'html.parser')
moviehref = str(divSoup.find("a")['href'])
print(moviehref)
movieURL = self.baseURL + moviehref
threadPool.putInQueue(parseMovie, {"movieURL": movieURL})
def parseMovie(movieURL):
movie = {}
moviePageHtml = urllib.request.urlopen(urllib.request.Request(movieURL, headers=self.hdr))
moviePageSoup = BeautifulSoup(moviePageHtml, 'html.parser')
movie['name'] = str(moviePageSoup.find('h2',{'class':'movie-original-title'}).string)
movie['duration'] = str(moviePageSoup.find('span',{'class':'running_time'}).string)
print(movie)
moviesVec.append(movie)
threadPool.putInQueue(checkNetflix, {'title': movie['name']})
def checkNetflix(title):
netflixWrapper = NetflixWrapper()
resp = netflixWrapper.isTitleInNetflix(title)
if(resp[0]):
netflixVec.append(resp[1])
for i in range(1, self.getWantToSeePages() + 1):
pageUrl = searchURL + '?pagina=' + str(i)
threadPool.putInQueue(parsePage, {'pageUrl': pageUrl})
#block until all tasks are done
threadPool.end()
return [moviesVec, netflixVec]
def __init__(self, server_addr, server_port):
self.server_port = server_port
self.s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.s.bind((server_addr, server_port))
self.pool = ThreadPool(10) #start with 10 threads
self.num_clients = 0
self.client_limit = CLIENT_LIMIT
#Dictionary used to map string representations of commands to functions which will perform the command
#Commands are functions listed in commands.py
self.command_map = {
"KILL_SERVICE": self.kill_service,
"HELO": self.echo,
"JOIN_CHATROOM": self.join_chatroom,
"LEAVE_CHATROOM": self.leave_chatroom,
"DISCONNECT": self.disconnect,
"CHAT": self.chat
}
self.chat_room_locks = []
self.chat_rooms = []
self.chat_room_map = {} #map a string name to integer index of the chatroom in chat_rooms
signal.signal(signal.SIGINT, exit_handler)
def _buildGivenPackages(self, listPackages, buildThreads):
# Extend listPackages from ["name1", "name2",..] to ["name1-vers1", "name2-vers2",..]
listPackageNamesAndVersions=set()
for pkg in listPackages:
base = SPECS.getData().getSpecName(pkg)
for version in SPECS.getData().getVersions(base):
listPackageNamesAndVersions.add(base+"-"+version)
returnVal = self._calculateParams(listPackageNamesAndVersions)
if not returnVal:
self.logger.error("Unable to set parameters. Terminating the package manager.")
raise Exception("Unable to set parameters")
statusEvent = threading.Event()
self._initializeScheduler(statusEvent)
self._initializeThreadPool(statusEvent)
for i in range(0, buildThreads):
workerName = "WorkerThread" + str(i)
ThreadPool.addWorkerThread(workerName)
ThreadPool.startWorkerThread(workerName)
statusEvent.wait()
Scheduler.stopScheduling = True
self.logger.debug("Waiting for all remaining worker threads")
ThreadPool.join_all()
setFailFlag = False
allPackagesBuilt = False
if Scheduler.isAnyPackagesFailedToBuild():
setFailFlag = True
if Scheduler.isAllPackagesBuilt():
allPackagesBuilt = True
if setFailFlag:
self.logger.error("Some of the packages failed:")
self.logger.error(Scheduler.listOfFailedPackages)
raise Exception("Failed during building package")
if not setFailFlag:
if allPackagesBuilt:
self.logger.debug("All packages built successfully")
else:
self.logger.error("Build stopped unexpectedly.Unknown error.")
raise Exception("Unknown error")
def verifAllProxy(self, pfile='Proxy.txt', out=None, threadnbr=15):
'''Verification de tous les proxys du fichier (pfile).
Peut enregistrer le Proxy verifier dans un fichier (out)
Methode a amelioré. '''
proxys = self.getProxysFromFile(pfile)
# def killthread(thread, timeout=30):
# """C'est pas beau !"""
# time.sleep(timeout)
# if thread.isAlive():
# thread._Thread__stop()
def verifT(prox):
if prox.verif():
self.goodproxy.append(prox)
pool = ThreadPool(threadnbr)
for prox in proxys:
pool.queueTask(verifT, prox)
timeout = (len(proxys) / threadnbr * 30 + 5) * 1.5
print 'Temps max : %s' % timeout
#time.sleep(timeout)
#print 'arret'
#print self.goodproxy
#TOTO mieux
count = 0
while pool.getThreadCount()>0:
time.sleep(1)
count+=1
if count>timeout:
pool.joinAll(False, False)
#print pool.getThreadCount()
if out != None:
with open(out, 'w') as pvfile:
for Proxy in self.goodproxy:
print str(Proxy)
pvfile.write(str(Proxy) + '\n')
def __init__(self, parent=None):
QThread.__init__(self, parent)
self.toDisplay = Queue()
self.threadpool = ThreadPool(max_workers=cpu_count())
def run_multi_integrate(xpar_init,inp_f=None,main_directory=None,
nThreads=2,init=0):
from ThreadPool import ThreadPool
# Copy
xpar = xpar_init.copy()
global FinishedThread
FinishedThread = 0
pool = ThreadPool(nThreads)
if init:
xpar.JOB = "INIT","DEFPIX","INTEGRATE","CORRECT"
files_to_copy = "XPARM.XDS","X-CORRECTIONS.pck","Y-CORRECTIONS.pck"
else:
xpar.JOB = "DEFPIX","INTEGRATE","CORRECT"
files_to_copy = "XPARM.XDS","BKGINIT.pck","BLANK.pck",\
"GAIN.pck","X-CORRECTIONS.pck","Y-CORRECTIONS.pck"
if not main_directory: main_directory = os.getcwd()
if os.path.isdir(main_directory):
os.chdir(main_directory)
else:
print "STOP! Directory not found:",directory
sys.exit()
range_list = new_range(xpar.DATA_RANGE, nThreads)
_templ = xpar.NAME_TEMPLATE_OF_DATA_FRAMES
if type(_templ) == type("") and _templ[0] != "/":
xpar.NAME_TEMPLATE_OF_DATA_FRAMES = "../" + \
xpar.NAME_TEMPLATE_OF_DATA_FRAMES
if type(_templ) == type(()) and _templ[0][0] != "/":
xpar.NAME_TEMPLATE_OF_DATA_FRAMES = "../" + \
xpar.NAME_TEMPLATE_OF_DATA_FRAMES[0], \
xpar.NAME_TEMPLATE_OF_DATA_FRAMES[1]
hklfiles = []
startTime = time()
print "\n"
for NTh in range(nThreads):
#newdir = os.path.join(main_directory,"integrate_batch_%d" % (NTh+1))
newdir = "integrate_batch_%d" % (NTh+1)
if not os.path.exists(newdir): os.mkdir(newdir)
for file in files_to_copy:
if os.path.isfile(file):
shutil.copyfile(file,os.path.join(newdir,file))
else:
print "STOP: Can't find file",file
sys.exit()
os.chdir(newdir)
xpar.DATA_RANGE = range_list[NTh]+1,range_list[NTh+1]
xpar.SPOT_RANGE = (range_list[NTh]+1,range_list[NTh]+4),
args = (xpar,inp_f,"xds.out",os.getcwd(),0)
pool.queueTask(run_xds_thread,args,taskCallback)
# To avoid chdir before the xds process is started...
print " ==> Threads XDS started for integration of images %4d to %4d"\
% (xpar.DATA_RANGE[0],xpar.DATA_RANGE[1])
sleep(2.0)
os.chdir("..")
hklfiles.append(os.path.join(newdir,"XDS_ASCII.HKL"))
pool.joinAll()
print "\n"
while FinishedThread != nThreads:
sleep(0.1)
#read cell
endTime = time()
print "\n Integration time: %.1f seconds" % (endTime - startTime)
H = read_xdsascii_head(hklfiles[0])
if H["friedels_law"] == "TRUE": H["friedels_law"] = 1
elif H["friedels_law"] == "FALSE": H["friedels_law"] = 0
dmin = get_maxResolution(os.path.join(newdir,"INTEGRATE.LP"))
xlatt = Lattice(H["cell"], "Unknown", symmetry=H["sym"],
dmin=dmin, friedels_law=H["friedels_law"])
run_xscale(hklfiles, "batch_merge.hkl", xlatt, save=1, out_f="xscale.out")
class TCPServer:
"""Server class. Creates, binds, and listens using server socket,
For every client that connects, its information is passed to a thread in a thread pool (see: ThreadPool.py)
"""
def __init__(self, server_addr, server_port):
self.server_port = server_port
self.s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.s.bind((server_addr, server_port))
self.pool = ThreadPool(10) #start with 10 threads
self.num_clients = 0
self.client_limit = CLIENT_LIMIT
#Dictionary used to map string representations of commands to functions which will perform the command
#Commands are functions listed in commands.py
self.command_map = {
"KILL_SERVICE": self.kill_service,
"HELO": self.echo,
"JOIN_CHATROOM": self.join_chatroom,
"LEAVE_CHATROOM": self.leave_chatroom,
"DISCONNECT": self.disconnect,
"CHAT": self.chat
}
self.chat_room_locks = []
self.chat_rooms = []
self.chat_room_map = {} #map a string name to integer index of the chatroom in chat_rooms
signal.signal(signal.SIGINT, exit_handler)
def listen(self):
"""Listen for connections and delegate to thread pool upon connection
"""
self.s.listen(5)
while True:
(client_s, client_addr) = self.s.accept()
print "SERVER_INFO: Client connected from: {0}".format(client_addr)
full = self.num_clients + 1 > self.client_limit
if full == True:
print "server full"
client_s.close()
else:
self.pool.execute(self.connection_handler, client_s, client_addr)
self.num_clients += 1
if self.num_clients > 30 and self.pool.num_threads < 25: #magic number as basic implementation
self.pool.add_threads(10)
elif self.num_clients < 20 and self.pool.num_threads > 35:
self.pool.remove_threads(10)
print "Clients: {0}".format(self.num_clients)
def connection_handler(self, socket, addr):
"""Function passed to thread pool which actively deals with clients"
Continuously receives messages, parses them into a command and command argument, and calls the
appropriate function using the dictionary defined above
"""
print "Connection handler"
s = socket
while True:
#if the connection is cut between server and client the server may hold an incorrect count of the current number of clients. this should be fixed by some protocol for determining if clients are connected (a heartbeat)
try:
msg_raw = s.recv(2048)
except SocketError: #s.close()
s.close()
self.num_clients -= 1
return False
else: #client just disconnected
if not msg_raw:
self.num_clients -= 1
s.close()
return False
cmd_arg_dict = utils.split_msg(msg_raw)
(cmd, arg) = cmd_arg_dict.items()[0]
print "CMD " + cmd
if cmd in self.command_map.keys():
self.command_map[cmd](socket=s, addr=addr, args=cmd_arg_dict, student_id=STUDENT_ID)
def kill_service(self, **kwargs):
print "SERVER_INFO: Kill command issued. Server shutting down....\n"
os.kill(os.getpid(), signal.SIGHUP)
def echo(self, **kwargs):
print "SERVER_INFO: Echo (HELO) command issued"
s = kwargs.get("socket", None)
(ip, port) = kwargs.get("addr", None)
args = kwargs.get("args", None)
return_string = args["HELO"]
student_id = kwargs.get("student_id", None)
reply = "HELO {0}\nIP:{1}\nPort:{2}\nStudentID:{3}\n".format(return_string, ip, port, student_id)
try:
s.send(reply)
except Exception:
print "SERVER_INFO: Error sending reply to client..."
def join_chatroom(self, **kwargs):
print "SERVER_MSG: join"
#for a join chatroom message we should see in the dictionary the keys: JOIN_CHATROOM (name), CLIENT_IP (0 for tcp), PORT (0 for tcp), CLIENT_NAME
#get args
args = kwargs["args"]
room_name = args["JOIN_CHATROOM"]
username = args["CLIENT_NAME"]
user_socket = kwargs["socket"]
#find the chatroom if it exists, or else create a new one
chat_room = None
room_ref = None
if room_name in self.chat_room_map:
room_ref = self.chat_room_map[room_name]
chat_room = self.chat_rooms[room_ref]
else:
room_ref = len(self.chat_rooms)
self.chat_room_map[room_name] = room_ref
chat_room = ChatRoom(room_name, room_ref) #pass the name, and identifier for chat room (array index)
self.chat_rooms.append(chat_room)
self.chat_room_locks.append(threading.RLock())
server_ip = utils.get_ip()
#add the user to the chat room
with self.chat_room_locks[room_ref]:
user_id = chat_room.add_user(username, user_socket)
user_socket.send("JOINED_CHATROOM: {0}\nSERVER_IP: {1}\nPORT: {2}\nROOM_REF: {3}\nJOIN_ID: {4}\n".format(room_name, server_ip, self.server_port, str(room_ref), str(user_id)))
chat_room.send_message("CHAT: {0}\nCLIENT_NAME: {1}\nMESSAGE: {2} has joined this chatroom.\n\n".format(str(room_ref), username, username))
def leave_chatroom(self, **kwargs):
print "SERVER_MSG: leave"
#for a join chatroom message we should see in the dictionary the keys: LEAVE_CHATROOM (name), CLIENT_NAME, JOIN_ID
#get args
args = kwargs["args"]
room_ref = int(args["LEAVE_CHATROOM"])
username = args["CLIENT_NAME"]
user_id = int(args["JOIN_ID"])
user_socket = kwargs["socket"]
chat_room = self.chat_rooms[room_ref]
user_socket.send("LEFT_CHATROOM: {0}\nJOIN_ID: {1}\n".format(room_ref, user_id))
with self.chat_room_locks[room_ref]:
chat_room.send_message("CHAT: {0}\nCLIENT_NAME: {1}\nMESSAGE: {2} has left this chatroom.\n\n".format(str(room_ref), username, username))
chat_room.remove_user(username)
def chat(self, **kwargs):
print "SERVER_MSG: chat"
##Keys for a chat message are: CHAT, JOIN_ID, CLIENT_NAME, MESSAGE
args = kwargs["args"]
room_ref = int(args["CHAT"])
username = args["CLIENT_NAME"]
user_id = int(args["JOIN_ID"])
message = args["MESSAGE"]
user_socket = kwargs["socket"]
chat_room = self.chat_rooms[room_ref]
chat_room.send_message("CHAT: {0}\nCLIENT_NAME: {1}\nMESSAGE: {2}\n\n".format(str(room_ref), username, message))
def disconnect(self, **kwargs):
print "SERVER_MSG: disconnect"
args = kwargs["args"]
username = args["CLIENT_NAME"]
user_socket = kwargs["socket"]
for i, room in enumerate(self.chat_rooms):
with self.chat_room_locks[i]:
if room.user_exists(username) == True:
room.send_message("CHAT: {0}\nCLIENT_NAME: {1}\nMESSAGE: {2} has left this chatroom.\n\n".format(str(i), username, username))
user_id = room.remove_user(username)
#user_socket.send("LEFT_CHATROOM: {0}\nJOIN_ID: {1}\n".format(str(i), str(user_id)))
self.num_clients -= 1
def runHPCEvent(self):
tolog("runHPCEvent")
self.__job.jobState = "running"
self.__job.setState([self.__job.jobState, 0, 0])
self.__job.pilotErrorDiag = None
rt = RunJobUtilities.updatePilotServer(self.__job, self.getPilotServer(), self.getPilotPort())
self.__JR.updateJobStateTest(self.__job, self.__jobSite, self.__node, mode="test")
defRes = self.getDefaultResources()
if defRes['copy_input_files'] == 'true':
self.__copyInputFiles = True
else:
self.__copyInputFiles = False
status, output, hpcJob = self.prepareHPCJob()
if status == 0:
tolog("HPC Job: %s " % hpcJob)
else:
tolog("failed to create the Tag file")
self.failJob(0, PilotErrors.ERR_UNKNOWN, self.__job, pilotErrorDiag=output)
return
self.__hpcStatus = None
self.__hpcLog = None
logFileName = None
tolog("runJobHPCEvent.getPilotLogFilename=%s"% self.getPilotLogFilename())
if self.getPilotLogFilename() != "":
logFileName = self.getPilotLogFilename()
hpcManager = HPCManager(globalWorkingDir=self.__job.workdir, logFileName=logFileName, poolFileCatalog=self.__poolFileCatalogTemp, inputFiles=self.__inputFilesGlobal, copyInputFiles=self.__copyInputFiles)
self.__hpcManager = hpcManager
self.HPCMode = "HPC_" + hpcManager.getMode(defRes)
self.__job.setMode(self.HPCMode)
self.__job.setHpcStatus('waitingResource')
rt = RunJobUtilities.updatePilotServer(self.__job, self.getPilotServer(), self.getPilotPort())
self.__JR.updatePandaServer(self.__job, self.__jobSite, self.__node, 25443)
hpcManager.getFreeResources(defRes)
self.__job.coreCount = hpcManager.getCoreCount()
self.__job.setHpcStatus('gettingEvents')
rt = RunJobUtilities.updatePilotServer(self.__job, self.getPilotServer(), self.getPilotPort())
self.__JR.updatePandaServer(self.__job, self.__jobSite, self.__node, 25443)
numRanges = hpcManager.getEventsNumber()
tolog("HPC Manager needs events: %s, max_events: %s; use the smallest one." % (numRanges, defRes['max_events']))
if numRanges > int(defRes['max_events']):
numRanges = int(defRes['max_events'])
eventRanges = self.getEventRanges(numRanges=numRanges)
#tolog("Event Ranges: %s " % eventRanges)
if len(eventRanges) == 0:
tolog("Get no Event ranges. return")
return
for eventRange in eventRanges:
self.__eventRanges[eventRange['eventRangeID']] = 'new'
# setup stage out
self.setupStageOutHPCEvent()
hpcManager.initJob(hpcJob)
hpcManager.initEventRanges(eventRanges)
hpcManager.submit()
threadpool = ThreadPool(defRes['stageout_threads'])
old_state = None
time_start = time.time()
while not hpcManager.isFinished():
state = hpcManager.poll()
self.__job.setHpcStatus(state)
if old_state is None or old_state != state or time.time() > (time_start + 60*10):
old_state = state
time_start = time.time()
tolog("HPCManager Job stat: %s" % state)
self.__JR.updateJobStateTest(self.__job, self.__jobSite, self.__node, mode="test")
rt = RunJobUtilities.updatePilotServer(self.__job, self.getPilotServer(), self.getPilotPort())
self.__JR.updatePandaServer(self.__job, self.__jobSite, self.__node, 25443)
if state and state == 'Complete':
break
outputs = hpcManager.getOutputs()
for output in outputs:
#self.stageOutHPCEvent(output)
threadpool.add_task(self.stageOutHPCEvent, output)
time.sleep(30)
self.updateHPCEventRanges()
tolog("HPCManager Job Finished")
self.__job.setHpcStatus('stagingOut')
rt = RunJobUtilities.updatePilotServer(self.__job, self.getPilotServer(), self.getPilotPort())
self.__JR.updatePandaServer(self.__job, self.__jobSite, self.__node, 25443)
outputs = hpcManager.getOutputs()
for output in outputs:
#self.stageOutHPCEvent(output)
threadpool.add_task(self.stageOutHPCEvent, output)
self.updateHPCEventRanges()
threadpool.wait_completion()
self.updateHPCEventRanges()
if len(self.__failedStageOuts) > 0:
tolog("HPC Stage out retry 1")
half_stageout_threads = defRes['stageout_threads'] / 2
if half_stageout_threads < 1:
half_stageout_threads = 1
threadpool = ThreadPool(half_stageout_threads)
failedStageOuts = self.__failedStageOuts
self.__failedStageOuts = []
for failedStageOut in failedStageOuts:
threadpool.add_task(self.stageOutHPCEvent, failedStageOut)
threadpool.wait_completion()
self.updateHPCEventRanges()
if len(self.__failedStageOuts) > 0:
tolog("HPC Stage out retry 2")
threadpool = ThreadPool(1)
failedStageOuts = self.__failedStageOuts
self.__failedStageOuts = []
for failedStageOut in failedStageOuts:
threadpool.add_task(self.stageOutHPCEvent, failedStageOut)
threadpool.wait_completion()
self.updateHPCEventRanges()
self.__job.setHpcStatus('finished')
self.__JR.updatePandaServer(self.__job, self.__jobSite, self.__node, 25443)
self.__hpcStatus, self.__hpcLog = hpcManager.checkHPCJobLog()
tolog("HPC job log status: %s, job log error: %s" % (self.__hpcStatus, self.__hpcLog))
class EventStager(object):
def __init__(self, workDir, setup, esPath, token, experiment, userid, sitename, outputDir=None, yodaToOS=False, threads=10, isDaemon=False, process=0, totalProcess=1):
self.__workDir = workDir
self.__updateEventRangesDir = os.path.join(self.__workDir, 'updateEventRanges_%s' % process)
if not os.path.exists(self.__updateEventRangesDir):
os.makedirs(self.__updateEventRangesDir)
self.__logFile = os.path.join(workDir, 'EventStager.log')
self.__setup = setup
self.__siteMover = S3ObjectstoreSiteMover(setup, useTimerCommand=False)
self.__esPath = esPath
self.__token = token
self.__experiment = experiment
self.__outputDir = outputDir
self.__userid = userid
self.__sitename = sitename
self.__report = getInitialTracingReport(userid=self.__userid, sitename=self.__sitename, dsname=None, eventType="objectstore", analysisJob=False, jobId=None, jobDefId=None, dn=self.__userid)
self.__num_stagingFile = 0
self.__eventRanges = {}
self.__eventRanges_staged = {}
self.__eventRanges_faileStaged = {}
self.__eventStager = None
self.__canFinish = False
self.__status = 'new'
self.__threads = threads
self.__isDaemon = isDaemon
self.__startTime = time.time()
self.__processedJobs = []
self.__handlingOthers = 0
self.__otherProcesses = []
self.__startWait = None
self.__waitTime = 15 * 60 # 15 minutes
self.__yodaToOS = yodaToOS
if not os.environ.has_key('PilotHomeDir'):
os.environ['PilotHomeDir'] = os.path.dirname(__file__)
self.__process = process
self.__totalProcess = totalProcess
self.__siteMover.setup(experiment)
self.__threadpool = ThreadPool(self.__threads)
logging.info("Init EventStager workDir %s setup %s esPath %s token %s experiment %s userid %s sitename %s threads %s outputDir %s isDaemond %s" % (self.__workDir, self.__setup, self.__esPath, self.__token, self.__experiment, self.__userid, self.__sitename, self.__threads, self.__outputDir, self.__isDaemon))
def renewEventStagerStatus(self):
canFinish_file = os.path.join(self.__workDir, 'EventStagerStatusCan.json')
finished_file = os.path.join(self.__workDir, 'EventStagerStatus.json')
if self.__isDaemon:
if os.path.exists(canFinish_file):
#with open(canFinish_file) as inputFile:
# origin_status = json.load(inputFile)
# self.__canFinish = origin_status['canFinish']
self.__canFinish = True
if self.__status == "finished":
status = {'status': self.__status}
with open(finished_file, 'w') as outputFile:
json.dump(status, outputFile)
elif os.path.exists(finished_file):
os.remove(finished_file)
else:
if os.path.exists(finished_file):
#with open(finished_file) as inputFile:
# origin_status = json.load(inputFile)
# self.__status = origin_status['status']
self.__status = "finished"
if self.__canFinish:
status = {'canFinish': self.__canFinish}
with open(canFinish_file, 'w') as outputFile:
json.dump(status, outputFile)
elif os.path.exists(canFinish_file):
os.remove(canFinish_file)
def start(self):
try:
self.renewEventStagerStatus()
if self.__outputDir:
stageCmd = "MVEventStager.py"
else:
stageCmd = "EventStager.py"
yoda_to_os = ''
if self.__yodaToOS:
yoda_to_os = '--YodaToOS '
if self.__setup and len(self.__setup.strip()):
cmd = 'python %s/%s --workDir %s --setup %s --esPath %s --token %s --experiment %s --userid %s --sitename %s --threads %s --outputDir %s %s--isDaemon 2>&1 1>>%s' % (self.__workDir, stageCmd, self.__workDir, self.__setup, self.__esPath, self.__token, self.__experiment, self.__userid, self.__sitename, self.__threads, self.__outputDir, yoda_to_os, self.__logFile)
else:
cmd = 'python %s/%s --workDir %s --esPath %s --token %s --experiment %s --userid %s --sitename %s --threads %s --outputDir %s %s--isDaemon 2>&1 1>>%s' % (self.__workDir, stageCmd, self.__workDir, self.__esPath, self.__token, self.__experiment, self.__userid, self.__sitename, self.__threads, self.__outputDir, yoda_to_os, self.__logFile)
pUtil.tolog("Start Event Stager: %s" % cmd)
self.__eventStager = subprocess.Popen(cmd, stdout=sys.stdout, stderr=sys.stdout, shell=True)
except:
pUtil.tolog("Failed to start Event Stager: %s" % traceback.format_exc())
def getLog(self):
return self.__logFile
def monitor(self):
try:
self.renewEventStagerStatus()
if not self.isFinished() and self.__eventStager is None or self.__eventStager.poll() is not None:
pUtil.tolog("Event Stager failed. Try to start it.")
self.start()
except:
pUtil.tolog("Failed to monitor Event Stager: %s" % traceback.format_exc())
def finish(self):
try:
pUtil.tolog("Tell Event Stager to finish after finishing staging out all events")
self.__canFinish = True
self.renewEventStagerStatus()
except:
pUtil.tolog("Failed to monitor Event Stager: %s" % traceback.format_exc())
def terminate(self):
try:
pUtil.tolog("Terminate Event Stager")
self.__eventStager.terminate()
except:
pUtil.tolog("Failed to terminate Event Stager: %s" % traceback.format_exc())
def isFinished(self):
if self.__canFinish and self.__status == 'finished':
return True
return False
def stageOutEvent(self, output_info):
filename, jobId, eventRangeID, status, output = output_info
try:
if status == 'failed':
self.__eventRanges_staged[filename].append((jobId, eventRangeID, status, output))
if eventRangeID not in self.__eventRanges[filename]:
logging.warning("stageOutEvent: %s is not in eventRanges" % eventRangeID)
else:
del self.__eventRanges[filename][eventRangeID]
if status == 'finished':
if not os.path.exists(output):
if eventRangeID in self.__eventRanges[filename]:
del self.__eventRanges[filename][eventRangeID]
return
ret_status, pilotErrorDiag, surl, size, checksum, self.arch_type = self.__siteMover.put_data(output, os.path.join(self.__esPath, os.path.basename(output)), lfn=os.path.basename(output), report=self.__report, token=self.__token, experiment=self.__experiment)
if ret_status == 0:
try:
self.__eventRanges_staged[filename].append((jobId, eventRangeID, status, output))
if eventRangeID not in self.__eventRanges[filename]:
logging.warning("stageOutEvent: %s is not in eventRanges" % eventRangeID)
else:
del self.__eventRanges[filename][eventRangeID]
#logging.info("Remove staged out output file: %s" % output)
#os.remove(output)
except Exception, e:
logging.info("!!WARNING!!2233!! remove ouput file threw an exception: %s" % (e))
else:
logging.info("!!WARNING!!1164!! Failed to upload file to objectstore: %d, %s" % (ret_status, pilotErrorDiag))
self.__eventRanges_faileStaged[filename].append((jobId, eventRangeID, status, output))
except:
logging.warning(traceback.format_exc())
self.__eventRanges_faileStaged[filename].append((jobId, eventRangeID, status, output))
def sort_file_by_mtime(self, path, files):
mtime = lambda f: os.stat(os.path.join(path, f)).st_mtime
return list(sorted(files, key=mtime))
def getUnstagedOutputFiles(self, ext=".dump"):
outputFiles = []
all_files = os.listdir(self.__workDir)
if ext == ".dump":
for file in all_files:
if file.endswith(ext):
if (int(hashlib.sha1(file).hexdigest(),16) % self.__totalProcess) == self.__process:
filename = os.path.join(self.__workDir, file)
outputFiles.append(file)
else:
for file in all_files:
if file.endswith(ext):
if self.__process == 0:
filename = os.path.join(self.__workDir, file)
outputFiles.append(file)
if outputFiles:
outputFiles = self.sort_file_by_mtime(self.__workDir, outputFiles)
logging.info("UnStaged Output files: %s" % outputFiles)
return outputFiles
def updateEventRange(self, event_range_id, status='finished'):
""" Update an event range on the Event Server """
pUtil.tolog("Updating an event range..")
message = ""
# url = "https://aipanda007.cern.ch:25443/server/panda"
url = "https://pandaserver.cern.ch:25443/server/panda"
node = {}
node['eventRangeID'] = event_range_id
# node['cpu'] = eventRangeList[1]
# node['wall'] = eventRangeList[2]
node['eventStatus'] = status
# tolog("node = %s" % str(node))
# open connection
ret = pUtil.httpConnect(node, url, path=self.__updateEventRangesDir, mode="UPDATEEVENTRANGE")
# response = ret[1]
if ret[0]: # non-zero return code
message = "Failed to update event range - error code = %d" % (ret[0])
else:
message = ""
return ret[0], message
def updateEventRanges(self, event_ranges):
""" Update an event range on the Event Server """
pUtil.tolog("Updating event ranges..")
message = ""
#url = "https://aipanda007.cern.ch:25443/server/panda"
url = "https://pandaserver.cern.ch:25443/server/panda"
# eventRanges = [{'eventRangeID': '4001396-1800223966-4426028-1-2', 'eventStatus':'running'}, {'eventRangeID': '4001396-1800223966-4426028-2-2','eventStatus':'running'}]
node={}
node['eventRanges']=json.dumps(event_ranges)
# open connection
ret = pUtil.httpConnect(node, url, path=self.__updateEventRangesDir, mode="UPDATEEVENTRANGES")
# response = json.loads(ret[1])
status = ret[0]
if ret[0]: # non-zero return code
message = "Failed to update event range - error code = %d, error: " % (ret[0], ret[1])
else:
response = json.loads(json.dumps(ret[1]))
status = int(response['StatusCode'])
message = json.dumps(response['Returns'])
return status, message
def cleanStagingFiles(self, older=None):
if older == None:
self.__eventRanges = {}
self.__eventRanges_staged = {}
self.__eventRanges_faileStaged = {}
all_files = os.listdir(self.__workDir)
for file in all_files:
if older == None:
if file.endswith(".dump.staging"):
origin_file = file.replace(".dump.staging", ".dump")
if (int(hashlib.sha1(origin_file).hexdigest(),16) % self.__totalProcess) == self.__process:
filepath = os.path.join(self.__workDir, file)
os.rename(filepath, filepath.replace(".dump.staging", ".dump"))
if file.endswith("cmdcopying"):
origin_file = file.replace("cmdcopying", "cmd")
if self.__process == 0:
filepath = os.path.join(self.__workDir, file)
os.rename(filepath, filepath.replace("cmdcopying", "cmd"))
else:
if file.endswith("cmdcopying"):
present = time.time()
origin_file = file.replace("cmdcopying", "cmd")
if self.__process == 0:
filepath = os.path.join(self.__workDir, file)
if (present - os.path.getmtime(filepath)) > older:
os.rename(filepath, filepath.replace("cmdcopying", "cmd"))
def getEventRanges(self):
if len(self.__eventRanges.keys()) > 5:
return
outputFiles = self.getUnstagedOutputFiles()
for file in outputFiles:
if len(self.__eventRanges.keys()) > 5:
return
self.__startWait = None
self.__eventRanges[file] = {}
self.__eventRanges_staged[file] = []
self.__eventRanges_faileStaged[file] = []
filepath = os.path.join(self.__workDir, file)
handle = open(filepath)
for line in handle:
if len(line.strip()) == 0:
continue
line = line.replace(" ", " ")
jobId, eventRange, status, output = line.split(" ")
output = output.split(",")[0]
self.__eventRanges[file][eventRange] = {'retry':0, 'event': (jobId, eventRange, status, output)}
self.__threadpool.add_task(self.stageOutEvent, (file, jobId, eventRange, status, output))
if jobId not in self.__processedJobs:
self.__processedJobs.append(jobId)
handle.close()
os.rename(filepath, filepath + ".staging")
def checkMissedStagingFiles(self):
all_files = os.listdir(self.__workDir)
for file in all_files:
try:
if file.endswith(".dump.staged.reported"):
origin_file = file.replace(".dump.staged.reported", ".dump")
filepath = os.path.join(self.__workDir, file)
size = os.path.getsize(filepath)
if size == 0:
if (int(hashlib.sha1(origin_file).hexdigest(),16) % self.__totalProcess) == self.__process:
back_file = filepath.replace(".dump.staged.reported", ".dump.BAK")
origin_file = filepath.replace(".dump.staged.reported", ".dump")
staging_file = filepath.replace(".dump.staged.reported", ".dump.staging")
if not os.path.exists(back_file) and not os.path.exists(origin_file) and not os.path.exists(staging_file):
os.remove(filepath)
os.rename(back_file, origin_file)
except:
logging.warning("Failed to rename %s to %s: %s" % (back_file, origin_file, traceback.format_exc()))
def checkFailedStagingFiles(self):
for file in self.__eventRanges_faileStaged:
while self.__eventRanges_faileStaged[file]:
jobId, eventRangeID, status, output = self.__eventRanges_faileStaged[file].pop()
if eventRangeID not in self.__eventRanges[file]:
logging.warning("checkFailedStagingFiles: %s is not in eventRanges" % eventRangeID)
else:
if self.__eventRanges[file][eventRangeID]['retry'] < 3:
self.__eventRanges[file][eventRangeID]['retry'] += 1
self.__threadpool.add_task(self.stageOutEvent, (file, jobId, eventRangeID, status, output))
else:
self.__eventRanges_staged[file].append((jobId, eventRangeID, 'failed', output))
del self.__eventRanges[file][eventRangeID]
def checkFinishedStagingFiles(self):
finishedFiles = []
for file in self.__eventRanges:
try:
if len(self.__eventRanges[file].keys()) == 0:
filepath = os.path.join(self.__workDir, file)
handle = open(filepath + ".staged.reported", 'w')
finishedEventRanges = []
for chunk in pUtil.chunks(self.__eventRanges_staged[file], 100):
try:
eventRanges = []
for outputEvents in chunk:
jobId, eventRangeID, status, output = outputEvents
if eventRangeID not in finishedEventRanges:
finishedEventRanges.append(eventRangeID)
if status == 'finished':
eventRanges.append({"eventRangeID": eventRangeID, "eventStatus": status})
if status.startswith("ERR"):
eventRanges.append({"eventRangeID": eventRangeID, "eventStatus": 'failed'})
update_status, update_output = self.updateEventRanges(eventRanges)
logging.info("update Event Range: status: %s, output: %s" % (update_status, update_output))
if update_status:
update_status, update_output = self.updateEventRanges(eventRanges)
logging.info("update Event retry Range: status: %s, output: %s" % (update_status, update_output))
if update_status == 0:
try:
ret_outputs = json.loads(json.loads(update_output))
if len(ret_outputs) == len(chunk):
for i in range(len(ret_outputs)):
try:
if ret_outputs[i]:
jobId, eventRangeID, status, output = chunk[i]
logging.info("Remove %s" % output)
os.remove(output)
handle.write('{0} {1} {2} {3}\n'.format(jobId, eventRangeID, status, output))
except:
logging.warning("Failed to remove %s: %s" % (output, traceback.format_exc()))
except:
logging.warning(traceback.format_exc())
except:
logging.warning(traceback.format_exc())
handle.close()
os.rename(filepath + ".staging", filepath + ".BAK")
finishedFiles.append(file)
except:
logging.warning(traceback.format_exc())
for file in finishedFiles:
del self.__eventRanges[file]
del self.__eventRanges_staged[file]
del self.__eventRanges_faileStaged[file]
def checkLostEvents(self):
for file in self.__eventRanges:
for eventRange in self.__eventRanges[file]:
jobId, eventRange, status, output = self.__eventRanges[file][eventRange]['event']
self.__threadpool.add_task(self.stageOutEvent, (file, jobId, eventRange, status, output))
def handleGfalFile(self, gfalFile):
try:
for i in range(3):
gfalFile = os.path.join(self.__workDir, gfalFile)
os.rename(gfalFile, gfalFile + "copying")
handle = open(gfalFile + "copying")
cmd = handle.read()
handle.close()
cmd = cmd.replace(" -t 3600 ", " -t 300 ")
logging.info("Execute command: %s" % cmd)
# status, output = commands.getstatusoutput(cmd)
status, output = TimerCommand(cmd).run(600)
logging.info("Status %s output %s" % (status, output))
if status == 0:
os.rename(gfalFile + "copying", gfalFile + "finished")
return
else:
os.rename(gfalFile + "copying", gfalFile)
os.rename(gfalFile, gfalFile + "failed")
except:
logging.error("handleGfalFile %s" % traceback.format_exc())
finally:
self.__handlingOthers -= 1
def handleS3File(self, s3File):
try:
s3File = os.path.join(self.__workDir, s3File)
os.rename(s3File, s3File + "copying")
handle = open(s3File + "copying")
cmd = handle.read()
handle.close()
source, destination = cmd.split(" ")
logging.info("S3 stage out from %s to %s" % (source, destination))
ret_status, pilotErrorDiag, surl, size, checksum, self.arch_type = self.__siteMover.put_data(source, destination, lfn=os.path.basename(destination), report=self.__report, token=self.__token, experiment=self.__experiment, timeout=300)
logging.info("Status %s output %s" % (ret_status, pilotErrorDiag))
if ret_status == 0:
os.rename(s3File + "copying", s3File + "finished")
else:
os.rename(s3File + "copying", s3File)
except:
logging.error("handleS3File %s" % traceback.format_exc())
finally:
self.__handlingOthers -= 1
def handleOtherFiles(self):
gfalFiles = self.getUnstagedOutputFiles(".gfalcmd")
for gfalFile in gfalFiles:
p = multiprocessing.Process(target=self.handleGfalFile, args=(gfalFile,))
p.start()
self.__otherProcesses.append(p)
self.__handlingOthers += 1
self.__startWait = None
s3Files = self.getUnstagedOutputFiles(".s3cmd")
for s3File in s3Files:
p = multiprocessing.Process(target=self.handleS3File, args=(s3File,))
p.start()
self.__otherProcesses.append(p)
self.__handlingOthers += 1
self.__startWait = None
termProcesses = []
for p in self.__otherProcesses:
if not p.is_alive():
termProcesses.append(p)
for p in termProcesses:
self.__otherProcesses.remove(p)
def killStallProcess(self):
command = "find /proc -maxdepth 1 -user wguan -type d -mmin +1 -exec basename {} \; | xargs ps | grep EventStager.py | awk '{ print $1 }' | grep -v " + str(os.getpid()) + "|xargs kill"
print command
status, output = commands.getstatusoutput(command)
print status
print output
def run(self):
logging.info("Start to run")
self.cleanStagingFiles()
timeStart = time.time() - 60
while not self.isFinished():
try:
if (time.time() - timeStart) > 60:
self.renewEventStagerStatus()
self.cleanStagingFiles(20*60)
# self.checkMissedStagingFiles()
self.getEventRanges()
self.checkFailedStagingFiles()
self.checkFinishedStagingFiles()
if self.__canFinish and len(self.__eventRanges.keys()) == 0:
self.__status = 'finished'
self.renewEventStagerStatus()
if self.__threadpool.is_empty():
self.checkLostEvents()
timeStart = time.time()
self.handleOtherFiles()
time.sleep(30)
logging.debug("len(eventranges:%s)" % len(self.__eventRanges.keys()))
#logging.debug("%s" % self.__eventRanges)
logging.debug("otherProcesses:%s" % len(self.__otherProcesses))
if len(self.__eventRanges.keys()) == 0 and len(self.__otherProcesses) == 0:
self.cleanStagingFiles()
if self.__startWait == None:
self.__startWait = time.time()
self.killStallProcess()
if self.__startWait and (time.time() - self.__startWait) > self.__waitTime:
break
except:
logging.info(traceback.format_exc())
#sys.exit(1)
logging.info("Finished to run")
self.pool = pool
def scan(self):
global PROGRESS_MAX
for root, dirs, files in walk(self.dir):
PROGRESS_MAX = len(files) * len (MODULES)
for fname in files:
self.__scanfile(os.path.join(root, fname))
def __scanfile(self, file):
scanner = FileScanner(file, self.outDir, self.modules, self.profile, self.pool)
scanner.scan()
if __name__ == '__main__':
pool = ThreadPool(MAX_THREADS)
if options.inputDir:
scanner = DirScanner(options.inputDir, options.outputDir, MODULES, options.profile, pool)
scanner.scan()
else:
scanner = FileScanner(options.inputFile, options.outputDir, MODULES, options.profile, pool)
scanner.scan()
pool.wait_completion()
#Search all results if required
if options.search:
if options.search == 'IP':
pattern = re.compile(r"\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b")
else:
def matchAll(phase, suffix):
global basics
global rules
global matches
global failures
global predicates
global locators
global noFiles
global noFilesAffected
global noUnits
global noPatternConstraints
global noPatternConstraintsOk
global noContentConstraints
global noContentConstraintsOk
global noPredicateConstraints
global noPredicateConstraintsOk
global noFragments
if (phase!="basics"):
basics = tools101.getBasics()
rules = json.load(open(const101.rulesDump, 'r'))["results"]["rules"]
matches = list()
failures = list()
predicates = set()
locators = set()
noFiles = 0
noUnits = 0
noFilesAffected = 0
noPatternConstraints = 0
noPatternConstraintsOk = 0
noContentConstraints = 0
noContentConstraintsOk = 0
noPredicateConstraints = 0
noPredicateConstraintsOk = 0
noFragments = 0
pool = ThreadPool(4)
print "Matching 101meta metadata on 101repo (phase \"" + str(phase)+ "\")."
for root, dirs, files in os.walk(os.path.join(const101.sRoot, "contributions"), followlinks=True):
if not root.startswith(os.path.join(const101.sRoot, ".git")+os.sep):
for basename in files:
noFiles += 1
if not basename in [".gitignore"]:
dirname = root[len(const101.sRoot)+1:]
pool.add_task(handleFile, phase, dirname, basename, suffix)
#handleFile(phase, dirname, basename, suffix)
sys.stdout.write('\n')
pool.wait_completion()
mr = dict()
mr["matches"] = matches
mr["failures"] = failures
mr["rules"] = rules
if phase=="predicates":
mr["predicates"] = list(predicates)
if phase=="fragments":
mr["locators"] = list(locators)
print str(noFiles) + " files examined."
print str(noFilesAffected) + " files affected."
print str(len(failures)) + " failures encountered."
print str(noUnits) + " metadata units attached."
print str(noContentConstraints) + " content constraints checked."
print str(noContentConstraintsOk) + " content constraints succeeded."
print str(noPatternConstraints) + " filename-pattern constraints checked."
print str(noPatternConstraintsOk) + " filename-pattern constraints succeeded."
if phase=="predicates":
print str(noPredicateConstraints) + " predicate constraints checked."
print str(noPredicateConstraintsOk) + " predicate constraints succeeded."
if phase=="fragments":
print str(len(locators)) + " fragment locators exercised."
print str(noFragments) + " fragment descriptions checked."
return mr
def handler( ips_cmds ):
taskCount = len( ips_cmds )
if taskCount > _THREAD_NUMBER:
threadCount = _THREAD_NUMBER
else:
threadCount = taskCount
pool=ThreadPool(threadCount)
for ip, cmds in ips_cmds:
pool.addTask(telnetTask, ip = ip, cmds = cmds )
pool.start()
dlg = wx.ProgressDialog("Waitng", "Waiting",
maximum = taskCount,
parent= None,
#style = wx.PD_CAN_ABORT|wx.PD_APP_MODAL
)
while pool.getFinishCount() < taskCount:
dlg.Update(pool.getFinishCount(), "%d of %d" %(pool.getFinishCount(), taskCount))
wx.MilliSleep(100)
dlg.Destroy()
result = pool.show()
pool.join()#pool itself is also a thread
return result
class DownloadServer():
def __init__(self, is_debug=False):
self.prev_day = {}
self.thread_pool = ThreadPool(
is_debug,
int(cfg.read('max_threads')),
int(cfg.read('max_buf')),
)
# 注意先清空先前留下的下载任务
# TODO: 改为继续执行先前未完成的任务
db.Execute("DELETE FROM `CurrentTask`")
def start(self):
# 首先启动线程池所有线程
self.thread_pool.start()
# 然后启动日历线程更新任务列表
start_new_thread(self.calendar_daemon, ())
# 同时启动生产者线程
start_new_thread(self.worker_daemon, ())
# 最后启动配额管理线程
start_new_thread(self.cleaner_daemon, ())
# 这个方法对于每个任务，判断是否进入了新的一天
# 如果是的话，就将新的任务增加到任务列表中
def update_calendar(self, overwrite_time=None):
# 注意同一个规则所实例化的任务，只能被添加一次
sql = "SELECT * FROM `UserTask` WHERE " \
"`TaskID` NOT IN (SELECT `TaskID` FROM `CurrentTask`) " \
"AND `Status` != 0"
this_day = {}
all_tasks = db.Query(sql)
for task in all_tasks:
# 首先读取时区信息，并转换为当前任务所在时区的时间
TimeZone = timezone(str(task[6]))
if overwrite_time is None:
today = datetime.now(TimeZone)
else:
today = overwrite_time
# 然后判断在该时区是否进入了新的一天
is_new_day = False
if self.prev_day.get(TimeZone, None) is None \
or today.day != self.prev_day[TimeZone]:
this_day[TimeZone] = today.day
is_new_day = True
# 如果确实进入了新的一天
if is_new_day:
# 首先生成任务开始和结束时间
# 同样注意转换为任务所在的时区
date_nums = map(int, task[5].split())
StartTime = datetime(year=today.year,
month=today.month,
day=today.day,
hour=date_nums[3],
minute=date_nums[4],
tzinfo=TimeZone)
FinishTime = StartTime + timedelta(hours=task[10])
# 生成一些与日期相关的数据
yesterday = today + timedelta(days=-1)
tomorrow = today + timedelta(days=1)
keywords = {
'%year%': today.year,
'%mon%': today.month,
'%day%': today.day,
'%prev_year%': yesterday.year,
'%prev_mon%': yesterday.month,
'%prev_day%': yesterday.day,
'%next_year%': tomorrow.year,
'%next_mon%': tomorrow.month,
'%next_day%': tomorrow.day
}
for key in keywords.keys():
keywords[key] = '%02d' % keywords[key]
# 其次生成下载链接
# 用dict中的关键字不断替换URL中字符串
TaskID = task[0]
UID = task[1]
URL = task[2]
for key in keywords.keys():
while URL.find(key) != -1:
URL = URL.replace(key, keywords[key])
# 生成URL后，更新文件保存位置：
# 1. 首先读取全局位置
Location = cfg.read('global_pos')
# 2. 然后定位到用户家目录位置
Location = os.path.join(Location, UID)
# 3. 再定位到规则目录位置，这里的类型都是unicode
RuleName = task[3]
SubDir = task[8]
if SubDir:
if type(SubDir) == str:
SubDir = SubDir.decode('utf-8')
Location = os.path.join(Location, SubDir)
# 4. 最后根据命名规则确定文件名
if task[9] == 'auto':
Location = os.path.join(Location, URL.split('/')[-1])
else:
Location = os.path.join(Location, RuleName + '.' + URL.split('.')[-1])
# 重新转换编码
if type(Location) == unicode:
Location = Location.encode('utf-8')
sql = "INSERT INTO `CurrentTask` VALUES ('%s', '%s', 1, '%s', '%s', '%s', %d, '%s', 0)" % (
UID, URL, Location, StartTime.ctime(), FinishTime.ctime(), TaskID, TimeZone.zone)
RepeatType = int(task[4])
if RepeatType == REP_PER_DAY:
# 如果是每天执行的任务，直接添加到任务列表
db.Execute(sql)
elif REP_PER_MON <= RepeatType <= REP_PER_SUN:
# 如果是周任务，则当前weekday必须匹配
if today.isoweekday() == RepeatType:
db.Execute(sql)
elif RepeatType == REP_PER_MONTH:
# 如果是月任务，日期必须匹配
if today.day == date_nums[2]:
db.Execute(sql)
elif RepeatType == REP_PER_YEAR:
# 如果是年任务，月日必须匹配
if today.month == date_nums[1] and today.day == date_nums[2]:
db.Execute(sql)
elif RepeatType == REP_PER_ONCE:
# 对于仅执行一次的任务，年月日必须匹配
# 并且放入任务列表中后就暂停掉这项任务
if today.year == date_nums[0] and today.month == date_nums[1] and today.day == date_nums[2]:
db.Execute(sql)
db.Execute("UPDATE `UserTask` SET `Status` = 0 WHERE `TaskID` = %d" % TaskID)
self.prev_day = this_day
# 这个方法定时检查任务列表
# 将过期的任务从任务列表中删除
# 将未过期的任务添加到下载线程池
# 任务被重试的次数越多，则下载优先级越低
def update_worker(self, overwrite_time=None):
# TODO: 添加任务前先检查当前网络是否连通
# 首先选择所有任务列表中未暂停且未被下载中的任务
sql = "SELECT * FROM `CurrentTask` WHERE `Status` = 1 ORDER BY `RepeatTimes` ASC"
all_task = db.Query(sql)
# 对于每一项任务进行处理，加入缓冲区
for task in all_task:
# 利用任务的时区信息，实例化两个时间戳
# 并且计算当前时刻在目标时区是几点
TimeZone = timezone(task[7])
if overwrite_time is None:
Now = datetime.now(TimeZone)
else:
Now = overwrite_time
StartTime = TimeZone.localize(parser.parse(task[4]))
FinishTime = TimeZone.localize(parser.parse(task[5]))
TaskID = task[6]
if Now > FinishTime:
# 如果任务已经超时，直接删除
sql = "DELETE FROM `CurrentTask` WHERE `TaskID` = %d" % TaskID
db.Execute(sql)
elif Now < StartTime:
# 如果该任务尚未开始，就继续处理下一项任务
continue
else:
# 如果这项任务应该被执行，就将其放入缓冲区
sql = "SELECT `Downloader`, `CheckType`, `CheckSize` FROM `UserTask` WHERE `TaskID` = %d" % TaskID
task_data = db.QueryFirst(sql)
data = {
'TaskID': TaskID,
'URL': task[1],
# 注意这里的编码，需要传入unicode
'Location': task[3].decode('utf-8'),
'Downloader': task_data[0],
'CheckType': task_data[1],
'CheckSize': task_data[2]
}
self.thread_pool.insert(data)
# 简而言之，实现一个文件配额功能
# 由于用户文件分目录保存，因此就不用新建数据库结构了
def clean_worker(self):
sql = "SELECT `UID`, `MaxSize`, `MaxFiles` FROM `Users`"
all_users = db.Query(sql)
base_dir = cfg.read('global_pos')
for user in all_users:
UID = user[0]
MaxSize = user[1]
MaxFiles = user[2]
user_home_dir = os.path.join(base_dir, UID)
TotalSize, TotalFiles = get_dir_size(user_home_dir)
TotalSize /= (1024 * 1024)
# 如果超出了文件数量配额或者文件大小配额
if TotalSize > MaxSize or TotalFiles > MaxFiles:
# 首先暂停该用户所有任务
sql = "UPDATE `UserTask` SET `Status` = 0 WHERE `UID` = '%s'" % UID
db.Execute(sql)
# 其次删除所有正在进行的任务
sql = "DELETE FROM `CurrentTask` WHERE `UID` = '%s'" % UID
db.Execute(sql)
# “生产者”函数的守护线程
def worker_daemon(self):
sys.stderr.write('Worker daemon of Download Server started!\n')
while True:
sleep(int(cfg.read('worker_checking_interval')))
self.update_worker()
# 维护任务列表的方法的守护线程
def calendar_daemon(self):
sys.stderr.write('Calendar daemon of Download Server started!\n')
while True:
self.update_calendar()
sleep(int(cfg.read('date_checking_interval')))
def cleaner_daemon(self):
while True:
sleep(int(cfg.read('cleaner_checking_interval')))
self.clean_worker()
def OnPatch(self,evt):
desc_ddr2 = "DDR2不需要修补补丁\r\n"
desc_fail = " 修补补丁失败\r\n"
desc_md5 = "补丁Md5值校验失败\r\n"
desc_notexist = "补丁文件不存在\r\n"
desc_success = "所有设备打补丁成功，请断电\r\n"
cmds_txt = "./cmds/cmds_patch.txt"
ips = self.ParseIp(self.ip.GetValue().encode('utf-8').split(';'))
if False == ips:
return
cmds = getContent(cmds_txt,"lines")
if False == cmds:
return
pool = ThreadPool( _THREAD_POOL_SIZE )
for ip in ips:
pool.addTask( telnetTask, ip = ip, cmds = cmds)
#必须放在pool.addFinishCb( finish_event,pool=pool)之前
def finish_event( *args, **kwargs ):
pool = kwargs['pool']
result = pool.show()
# [[ip,[ret1,ret2]],...]
result_to_show = ""
for ip_retlist in result:
ip = ip_retlist[0]
retlist = ip_retlist[1]
if ["fail"] == retlist:
result_to_show += ip + " Telnet时抛出了异常\r\n"
else:
t = retlist[len(retlist)-1].split("\r\n")
status = t[len(t)-2]
if "0" == status:
pass
elif "1" == status:
result_to_show += ip + desc_ddr2
elif "2" == status:
result_to_show += ip + desc_md5
elif "3" == status:
result_to_show += ip + desc_notexist
elif "4" == status:
result_to_show += ip + desc_fail
else:
result_to_show += ip + " 脚本返回意料之外的值\r\n"
last_cmd_result = retlist[len(retlist)-1]
logging.warning(toUnicode(last_cmd_result))
if "" == result_to_show:
result_to_show = desc_success
if "请断电" in desc_success:
wx.CallAfter(pub.sendMessage,"importNote",("请将设备断电"))
logging.warning(result_to_show.decode('utf-8'))
result_to_show = self.result_text.GetValue()+getCurTime().decode("ascii")+" ".decode("utf-8")+result_to_show.decode("utf-8")
self.result_text.SetValue(result_to_show)
self.ButtonStatus("Enable")
pool.addFinishCb( finish_event,pool=pool)
pool.start()
self.ButtonStatus("Disable")
