def __init__(self, manager, data, pid):
        """constructor"""
        BaseThread.__init__(self, manager)
        self.data = data
        self.pid = pid

        self.start()
Exemple #2
0
 def __init__(self, master, cond):
     BaseThread.__init__(self, 'Handler')
     self.master = master
     self.message_queue = Queue.Queue()
     self.uuid_queue = Queue.Queue()
     self.cond = cond
     self.sleep_flag = False
Exemple #3
0
    def __init__(self, id, workagent, cond, name=None, worker_class=None):
        if not name:
            name = "worker_%s" % id
        BaseThread.__init__(self, name)
        self.workeragent = workagent
        self.id = id
        self.running_task = None  #TASK obj
        self.cond = cond
        self.initialized = False
        self.setup_flag = True
        self.finialized = False
        self.fin_flag = False
        self.status = WorkerStatus.NEW

        self.finish_task = None

        global wlog
        self.log = wlog

        self.worker_obj = None
        if worker_class:
            self.worker_obj = worker_class(self.log)
            self.log.debug('[Worker_%s] Create Worker object %s' %
                           (self.id, self.worker_obj.__class__.__name__))
        #self.proc_log = open("%s/worker_%d.log"%(self.workeragent.cfg.getCFGattr("Rundir"),self.id),'w+')
        self.log.debug(
            '[Worker_%s] Worker Process log path:%s/worker_%d.log' %
            (self.id, self.workeragent.cfg.getCFGattr("Rundir"), self.id))

        self.process = None
        self.recode = 0
Exemple #4
0
    def __init__(self, manager, data, pid):
        """constructor"""
        BaseThread.__init__(self, manager)
        self.data = data
        self.pid = pid

        self.start()
Exemple #5
0
    def __init__(self, manager, data, pid):
        # TODO: owner
        BaseThread.__init__(self, manager)
        # [... (plugin, url) ...]
        self.data = data
        self.pid = pid

        self.start()
Exemple #6
0
 def __init__(self, master, appmgr):
     BaseThread.__init__(self, name=self.__class__.__name__)
     self.master = master
     self.appmgr = appmgr
     self.task_todo_Queue = Queue.Queue()
     self.completed_Queue = Queue.Queue()
     #self.task_unschedule_queue = Queue.Queue()
     self.policy = Policy.Policy()
Exemple #7
0
    def __init__(self, manager, data, pid):
        # TODO: owner
        BaseThread.__init__(self, manager)
        # [... (plugin, url) ...]
        self.data = data
        self.pid = pid

        self.start()
Exemple #8
0
    def __init__(self, manager):
        """Constructor"""
        BaseThread.__init__(self, manager)

        self.queue = Queue()  # job queue
        self.active = None

        self.start()
    def __init__(self, manager):
        """Constructor"""
        BaseThread.__init__(self, manager)

        self.queue = Queue() # job queue
        self.active = None

        self.start()
Exemple #10
0
 def __init__(self, callback, func, *args, **kwds):
     BaseThread.__init__(self,'async_call_%s'%func.__name__)
     self.callback = callback
     self.func = func
     self.args = args
     self.kwds = kwds
     self.result = None
     self.exception = None
     self.start_time = 0
     self.stop_time = 0
Exemple #11
0
    def __init__(self, workagent, cond):
        BaseThread.__init__(self, "worker")
        self.workagent = workagent
        self.running_task = None

        self.cond = cond

        self.initialized = False
        self.finialize = False
        self.status = WorkerStatus.NEW
Exemple #12
0
    def __init__(self, manager, owner, data, pid=-1, oc=None):
        BaseThread.__init__(self, manager, owner)

        # [... (plugin, url) ...]
        self.data = data
        self.pid = pid
        self.oc = oc # online check
        # urls that already have a package name
        self.names = {}

        self.start()
Exemple #13
0
    def __init__(self, manager, owner, data, pid=-1, oc=None):
        BaseThread.__init__(self, manager, owner)

        # [... (plugin, url) ...]
        self.data = data
        self.pid = pid
        self.oc = oc  # online check
        # urls that already have a package name
        self.names = {}

        self.start()
Exemple #14
0
    def __init__(self, manager):
        """Constructor"""
        BaseThread.__init__(self, manager)

        self.isWorking = Event()
        self.isWorking.clear()

        self.queue = Queue()  # job queue
        self.active = None

        self.start()
Exemple #15
0
    def __init__(self, manager, data, pid):
        # TODO: owner
        BaseThread.__init__(self, manager)
        # [... (plugin, url) ...]
        self.data = data
        self.pid = pid
        # holds the progress, while running
        self.progress = None

        self.m.addThread(self)
        self.start()
Exemple #16
0
    def __init__(self, manager):
        """Constructor"""
        BaseThread.__init__(self, manager)

        self.isWorking = Event()
        self.isWorking.clear()

        self.queue = Queue()  # job queue
        self.active = None

        self.start()
Exemple #17
0
 def __init__(self, client, worker_agent, cond):
     BaseThread.__init__(self, name='HeartbeatThread')
     self._client = client
     self.worker_agent = worker_agent
     self.queue_lock = threading.RLock()
     self.acquire_queue = Queue.Queue()  # entry = key:val
     self.interval = Config.Config.getCFGattr(
         'HeartBeatInterval') if Config.Config.getCFGattr(
             'HeartBeatInterval') else 0.1
     self.cond = cond
     global wlog
    def __init__(self, manager, data, fid, pid, owner):
        BaseThread.__init__(self, manager, owner)
        # [... (url, plugin) ...]
        self.data = data
        self.fid = fid
        self.pid = pid
        # holds the progress, while running
        self.progress = None
        # holds if an error happened
        self.error = False

        self.start()
Exemple #19
0
    def __init__(self, manager, data, fid, pid, owner):
        BaseThread.__init__(self, manager, owner)
        # [... (url, plugin) ...]
        self.data = data
        self.fid = fid
        self.pid = pid
        # holds the progress, while running
        self.progress = None
        # holds if an error happened
        self.error = False

        self.start()
Exemple #20
0
    def __init__(self, m, function, args, kwargs):
        """Constructor"""
        BaseThread.__init__(self, m)

        self.f = function
        self.args = args
        self.kwargs = kwargs

        self.active = []

        m.localThreads.append(self)

        self.start()
Exemple #21
0
    def __init__(self, manager, data, pid=-1, rid=-1):
        """Constructor"""
        BaseThread.__init__(self, manager)

        self.data = data
        self.pid = pid  # package id
        # [ .. (name, plugin) .. ]

        self.rid = rid  #result id

        self.cache = []  #accumulated data

        self.start()
Exemple #22
0
    def __init__(self, m, function, args, kwargs):
        """Constructor"""
        BaseThread.__init__(self, m)

        self.f = function
        self.args = args
        self.kwargs = kwargs

        self.active = []

        m.localThreads.append(self)

        self.start()
    def __init__(self, manager, data, pid=-1, rid=-1):
        """Constructor"""
        BaseThread.__init__(self, manager)

        self.data = data
        self.pid = pid # package id
        # [ .. (name, plugin) .. ]

        self.rid = rid #result id

        self.cache = [] #accumulated data

        self.start()
Exemple #24
0
    def __init__(self, m, function, args, kwargs):
        """Constructor"""
        BaseThread.__init__(self, m)

        self.f = function
        self.args = args
        self.kwargs = kwargs

        self.active = []
        self.progress = 0

        m.addThread(self)

        self.start()
Exemple #25
0
    def __init__(self, mixer, sleep=False, max_client=None, save=False):
        Parent.__init__(self)
        self.mixer = mixer
        self.save_file = save
        self.max_client = max_client
        self.sleep = sleep
        self.file_index = 0

        self._voice_ids = [
            'Geraint', 'Gwyneth', 'Mads', 'Naja', 'Hans', 'Marlene', 'Nicole',
            'Russell', 'Amy', 'Brian', 'Emma', 'Raveena', 'Ivy', 'Joanna',
            'Joey', 'Justin', 'Kendra', 'Kimberly', 'Salli', 'Conchita',
            'Enrique', 'Miguel', 'Penelope', 'Chantal', 'Celine', 'Mathieu',
            'Dora', 'Karl', 'Carla', 'Giorgio', 'Mizuki', 'Liv', 'Lotte',
            'Ruben', 'Ewa', 'Jacek', 'Jan', 'Maja', 'Ricardo', 'Vitoria',
            'Cristiano', 'Ines', 'Carmen', 'Maxim', 'Tatyana', 'Astrid',
            'Filiz', 'Vicki'
        ]
Exemple #26
0
# 多執行緒的前工作
def my_thread_job():
    with sem:
        print("work")


# 多執行緒的後工作
def cb(year, month):
    with sem:
        fileName = 'NYC_{}_{}.json'.format(year, month)
        filePath = 'NYC_{}_{}.json'.format(year, month)
        monthData = nYC.get_monthData(year, month)
        nYC.write_monthData(fileName, monthData)
        data = nYC.openJson(fileName)
        print('{},{},{}'.format(year, month, data['response']['meta']['hits']))
        print('{},{},{}'.format(year, month, len(data['response']['docs'])))


# testing

nYC = newYorkCrawl()
year = sys.argv[1]
maxMonth = sys.argv[2]
maxMonth = int(maxMonth)

sem = threading.Semaphore(4)
for month in range(1, maxMonth):
    BaseThread(name='test',
               target=my_thread_job,
               callback=cb,
               callback_args=(year, month)).start()
            append_data(append_fileName, data)
        except:
            with open("error_content", 'a+', encoding='utf8') as f:
                f.write(url)
                f.write("\n")
            print("error")
        else:
            print("{} done".format(childNumber))


nYC = newYorkCrawl()
year = sys.argv[1]
month = sys.argv[2]

fileName = "NYC_{}_{}.json".format(year, month)
append_fileName = "NYC_{}_{}.txt".format(year, month)

data = nYC.openJson(fileName)
print('{},{},{}'.format(year, month, data['response']['meta']['hits']))
print('{},{},{}'.format(year, month, len(data['response']['docs'])))
maxItem = len(data['response']['docs'])

sem = threading.Semaphore(100)
for child in range(0, maxItem):
    if child != 0 and child % 1000 == 0:
        time.sleep(200)
    tmp = data['response']['docs'][child]
    BaseThread(name='test',
               target=my_thread_job,
               callback=cb,
               callback_args=(tmp, child)).start()
from BaseThread import BaseThread
import threading
import time

# 多執行緒的前工作
def my_thread_job():
    with sem:
        print("{} runing".format("hi"))
        time.sleep(1)
# 多執行緒的後工作
def cb(argv1, argv2):
    with sem:
        print("{} {}".format(argv1, argv2))


sem=threading.Semaphore(4)

for i in range(5):
    BaseThread(
    name = 'test',
    target=my_thread_job,
    callback=cb,
    callback_args=("hello","word")
    ).start()
Exemple #29
0
 def __init__(self, master):
     BaseThread.__init__(self, 'ControlThread')
     self.master = master
     self.processing = False
Exemple #30
0
 def __init__(self, max_images):
     Parent.__init__(self)
     self.max_images = max_images
     self.pool = ThreadPool(10)
Exemple #31
0
 def __init__(self, client, worker_agent):
     BaseThread.__init__(self, name='HeartbeatThread')
     self._client = client
     self.worker_agent = worker_agent
Exemple #32
0
 def __init__(self, master):
     BaseThread.__init__(self, 'WatchDog')
     self.master = master
     self.processing = False