def __init__(self, manager, data, pid):
"""constructor"""
BaseThread.__init__(self, manager)
self.data = data
self.pid = pid
self.start()
def __init__(self, master, cond):
BaseThread.__init__(self, 'Handler')
self.master = master
self.message_queue = Queue.Queue()
self.uuid_queue = Queue.Queue()
self.cond = cond
self.sleep_flag = False
def __init__(self, id, workagent, cond, name=None, worker_class=None):
if not name:
name = "worker_%s" % id
BaseThread.__init__(self, name)
self.workeragent = workagent
self.id = id
self.running_task = None #TASK obj
self.cond = cond
self.initialized = False
self.setup_flag = True
self.finialized = False
self.fin_flag = False
self.status = WorkerStatus.NEW
self.finish_task = None
global wlog
self.log = wlog
self.worker_obj = None
if worker_class:
self.worker_obj = worker_class(self.log)
self.log.debug('[Worker_%s] Create Worker object %s' %
(self.id, self.worker_obj.__class__.__name__))
#self.proc_log = open("%s/worker_%d.log"%(self.workeragent.cfg.getCFGattr("Rundir"),self.id),'w+')
self.log.debug(
'[Worker_%s] Worker Process log path:%s/worker_%d.log' %
(self.id, self.workeragent.cfg.getCFGattr("Rundir"), self.id))
self.process = None
self.recode = 0
def __init__(self, manager, data, pid):
"""constructor"""
BaseThread.__init__(self, manager)
self.data = data
self.pid = pid
self.start()
def __init__(self, manager, data, pid):
# TODO: owner
BaseThread.__init__(self, manager)
# [... (plugin, url) ...]
self.data = data
self.pid = pid
self.start()
def __init__(self, master, appmgr):
BaseThread.__init__(self, name=self.__class__.__name__)
self.master = master
self.appmgr = appmgr
self.task_todo_Queue = Queue.Queue()
self.completed_Queue = Queue.Queue()
#self.task_unschedule_queue = Queue.Queue()
self.policy = Policy.Policy()
def __init__(self, manager, data, pid):
# TODO: owner
BaseThread.__init__(self, manager)
# [... (plugin, url) ...]
self.data = data
self.pid = pid
self.start()
def __init__(self, manager):
"""Constructor"""
BaseThread.__init__(self, manager)
self.queue = Queue() # job queue
self.active = None
self.start()
def __init__(self, manager):
"""Constructor"""
BaseThread.__init__(self, manager)
self.queue = Queue() # job queue
self.active = None
self.start()
def __init__(self, callback, func, *args, **kwds):
BaseThread.__init__(self,'async_call_%s'%func.__name__)
self.callback = callback
self.func = func
self.args = args
self.kwds = kwds
self.result = None
self.exception = None
self.start_time = 0
self.stop_time = 0
def __init__(self, workagent, cond):
BaseThread.__init__(self, "worker")
self.workagent = workagent
self.running_task = None
self.cond = cond
self.initialized = False
self.finialize = False
self.status = WorkerStatus.NEW
def __init__(self, manager, owner, data, pid=-1, oc=None):
BaseThread.__init__(self, manager, owner)
# [... (plugin, url) ...]
self.data = data
self.pid = pid
self.oc = oc # online check
# urls that already have a package name
self.names = {}
self.start()
def __init__(self, manager, owner, data, pid=-1, oc=None):
BaseThread.__init__(self, manager, owner)
# [... (plugin, url) ...]
self.data = data
self.pid = pid
self.oc = oc # online check
# urls that already have a package name
self.names = {}
self.start()
def __init__(self, manager):
"""Constructor"""
BaseThread.__init__(self, manager)
self.isWorking = Event()
self.isWorking.clear()
self.queue = Queue() # job queue
self.active = None
self.start()
def __init__(self, manager, data, pid):
# TODO: owner
BaseThread.__init__(self, manager)
# [... (plugin, url) ...]
self.data = data
self.pid = pid
# holds the progress, while running
self.progress = None
self.m.addThread(self)
self.start()
def __init__(self, manager):
"""Constructor"""
BaseThread.__init__(self, manager)
self.isWorking = Event()
self.isWorking.clear()
self.queue = Queue() # job queue
self.active = None
self.start()
def __init__(self, client, worker_agent, cond):
BaseThread.__init__(self, name='HeartbeatThread')
self._client = client
self.worker_agent = worker_agent
self.queue_lock = threading.RLock()
self.acquire_queue = Queue.Queue() # entry = key:val
self.interval = Config.Config.getCFGattr(
'HeartBeatInterval') if Config.Config.getCFGattr(
'HeartBeatInterval') else 0.1
self.cond = cond
global wlog
def __init__(self, manager, data, fid, pid, owner):
BaseThread.__init__(self, manager, owner)
# [... (url, plugin) ...]
self.data = data
self.fid = fid
self.pid = pid
# holds the progress, while running
self.progress = None
# holds if an error happened
self.error = False
self.start()
def __init__(self, manager, data, fid, pid, owner):
BaseThread.__init__(self, manager, owner)
# [... (url, plugin) ...]
self.data = data
self.fid = fid
self.pid = pid
# holds the progress, while running
self.progress = None
# holds if an error happened
self.error = False
self.start()
def __init__(self, m, function, args, kwargs):
"""Constructor"""
BaseThread.__init__(self, m)
self.f = function
self.args = args
self.kwargs = kwargs
self.active = []
m.localThreads.append(self)
self.start()
def __init__(self, manager, data, pid=-1, rid=-1):
"""Constructor"""
BaseThread.__init__(self, manager)
self.data = data
self.pid = pid # package id
# [ .. (name, plugin) .. ]
self.rid = rid #result id
self.cache = [] #accumulated data
self.start()
def __init__(self, m, function, args, kwargs):
"""Constructor"""
BaseThread.__init__(self, m)
self.f = function
self.args = args
self.kwargs = kwargs
self.active = []
m.localThreads.append(self)
self.start()
def __init__(self, manager, data, pid=-1, rid=-1):
"""Constructor"""
BaseThread.__init__(self, manager)
self.data = data
self.pid = pid # package id
# [ .. (name, plugin) .. ]
self.rid = rid #result id
self.cache = [] #accumulated data
self.start()
def __init__(self, m, function, args, kwargs):
"""Constructor"""
BaseThread.__init__(self, m)
self.f = function
self.args = args
self.kwargs = kwargs
self.active = []
self.progress = 0
m.addThread(self)
self.start()
def __init__(self, mixer, sleep=False, max_client=None, save=False):
Parent.__init__(self)
self.mixer = mixer
self.save_file = save
self.max_client = max_client
self.sleep = sleep
self.file_index = 0
self._voice_ids = [
'Geraint', 'Gwyneth', 'Mads', 'Naja', 'Hans', 'Marlene', 'Nicole',
'Russell', 'Amy', 'Brian', 'Emma', 'Raveena', 'Ivy', 'Joanna',
'Joey', 'Justin', 'Kendra', 'Kimberly', 'Salli', 'Conchita',
'Enrique', 'Miguel', 'Penelope', 'Chantal', 'Celine', 'Mathieu',
'Dora', 'Karl', 'Carla', 'Giorgio', 'Mizuki', 'Liv', 'Lotte',
'Ruben', 'Ewa', 'Jacek', 'Jan', 'Maja', 'Ricardo', 'Vitoria',
'Cristiano', 'Ines', 'Carmen', 'Maxim', 'Tatyana', 'Astrid',
'Filiz', 'Vicki'
]
# 多執行緒的前工作
def my_thread_job():
with sem:
print("work")
# 多執行緒的後工作
def cb(year, month):
with sem:
fileName = 'NYC_{}_{}.json'.format(year, month)
filePath = 'NYC_{}_{}.json'.format(year, month)
monthData = nYC.get_monthData(year, month)
nYC.write_monthData(fileName, monthData)
data = nYC.openJson(fileName)
print('{},{},{}'.format(year, month, data['response']['meta']['hits']))
print('{},{},{}'.format(year, month, len(data['response']['docs'])))
# testing
nYC = newYorkCrawl()
year = sys.argv[1]
maxMonth = sys.argv[2]
maxMonth = int(maxMonth)
sem = threading.Semaphore(4)
for month in range(1, maxMonth):
BaseThread(name='test',
target=my_thread_job,
callback=cb,
callback_args=(year, month)).start()
append_data(append_fileName, data)
except:
with open("error_content", 'a+', encoding='utf8') as f:
f.write(url)
f.write("\n")
print("error")
else:
print("{} done".format(childNumber))
nYC = newYorkCrawl()
year = sys.argv[1]
month = sys.argv[2]
fileName = "NYC_{}_{}.json".format(year, month)
append_fileName = "NYC_{}_{}.txt".format(year, month)
data = nYC.openJson(fileName)
print('{},{},{}'.format(year, month, data['response']['meta']['hits']))
print('{},{},{}'.format(year, month, len(data['response']['docs'])))
maxItem = len(data['response']['docs'])
sem = threading.Semaphore(100)
for child in range(0, maxItem):
if child != 0 and child % 1000 == 0:
time.sleep(200)
tmp = data['response']['docs'][child]
BaseThread(name='test',
target=my_thread_job,
callback=cb,
callback_args=(tmp, child)).start()
from BaseThread import BaseThread
import threading
import time
# 多執行緒的前工作
def my_thread_job():
with sem:
print("{} runing".format("hi"))
time.sleep(1)
# 多執行緒的後工作
def cb(argv1, argv2):
with sem:
print("{} {}".format(argv1, argv2))
sem=threading.Semaphore(4)
for i in range(5):
BaseThread(
name = 'test',
target=my_thread_job,
callback=cb,
callback_args=("hello","word")
).start()
def __init__(self, master):
BaseThread.__init__(self, 'ControlThread')
self.master = master
self.processing = False
def __init__(self, max_images):
Parent.__init__(self)
self.max_images = max_images
self.pool = ThreadPool(10)
def __init__(self, client, worker_agent):
BaseThread.__init__(self, name='HeartbeatThread')
self._client = client
self.worker_agent = worker_agent
def __init__(self, master):
BaseThread.__init__(self, 'WatchDog')
self.master = master
self.processing = False