def start(self): self._task_fetcher.start() #self._task_fetcher.join() self._output_pusher.start() #self._output_pusher.join() for worker in self._workers: worker.start()
def dummy(mission, thread): """After download, continue next mission""" if thread is self.download_thread: mission = self.get_mission_to_download() if mission: worker = self.create_child(download) worker.start(mission, setting["savepath"]) self.download_thread = worker else: safeprint("All download completed")
def handler(event, context, upload_file=True): _set_env() tasks = [] for record in event['Records']: bucket = record['s3']['bucket']['name'] key = record['s3']['object']['key'] key = urllib.unquote(key) tasks.append((bucket, key)) worker.start(tasks, upload_file)
def start_work(self): #download if self.mode == 0: self.file_data = [[] for i in self.clients] #upload else: self.file_data = fileops.read_file(self.filename, self.bandwidth_ratio, self.buff_size) #Intiate workers for worker in self.workers: worker.start()
#!/usr/bin/env python # -*- coding: utf-8 -*- from configmgr import configmgr from singleton import singleton import worker import testroutine @singleton class sometestcls: def __init__(self): pass if __name__ == "__main__": configmgr().read('./config.json') testroutine.run() worker.start()
''' Created on 2013-2-16 @author: Administrator ''' import worker import multiprocessing class SomeObject(): def __init__(self,name,sex): self._name=name self._sex=sex def say(self): print('%s is %s old' %(self._name,self._sex)) if __name__=='__main__': obj=SomeObject('zhang','30') queue=multiprocessing.Queue() worker=worker.Worker(queue) worker.start() queue.put(obj) print ('in main process %d' %(multiprocessing.current_process().pid)) worker.join()
def initparam(): """ 初始化参数 """ params = OrderedDict() print "【从前端获得参数】 %s" % str(request.values) print "【正在处理参数】" # 【开始时间】 params["start_time"] = request.form.get("startTime", type=str) if is_timestr(params['start_time']) is False: params['start_time'] = get_time() print "\t[开始时间] %s" % params["start_time"] # 【结束时间】 params["end_time"] = request.form.get("endTime", type=str) if is_timestr(params['end_time']) is False: params['end_time'] = "未设置" # 不结束 print "\t[结束时间] %s" %params["end_time"] # 【时间间隔】 params["interval"] = request.form.get("interval", type=int) params['step'] = params['interval'] / 5 # 步长 print "\t[时间间隔] %d" % params["interval"] print "\t[保存图片的步长] %d" % params["step"] # 【处理点】 points = { 'center_point' : request.form.get("centerPoint", type=str), 'north_west_point' : request.form.get("northWestPoint", type=str), 'north_east_point' : request.form.get("northEastPoint", type=str), 'south_east_point' : request.form.get("southEastPoint", type=str), 'south_west_point' : request.form.get("southWestPoint", type=str) } print '\t[点] %s' % str(points) for point_name,value in points.items(): params[point_name] = get_latlng( value ) # 【保存文件夹】 params["save_file_dir"] = request.form.get("saveFileDir", type=str) print "\t[保存文件夹] %s" % params["save_file_dir"] save_dir = params["save_file_dir"] if not os.path.exists(save_dir): # 选择的文件夹不存在 print "\t【WARNING】 选择的文件夹不存在,跳转到初始页面" print "【params】" + str(params) return redirect_index("文件夹路径不存在,请重新输入!") # # 【输出文件夹】 out_dir = os.path.join(save_dir, params['start_time']) params['out_dir'] = out_dir print "\t[图像输出文件夹] %s" % params["out_dir"] if os.path.exists(out_dir): # 输出文件夹已经存在 print "【params】" + str(params) return redirect_index("输出文件夹已经存在,%s<Br/>请重新输入文件夹!" % out_dir) os.makedirs(out_dir) # 项目备注 params["remark"] = request.form.get("remark") print "\t[项目备注] %s" % params["remark"] img_len , request_points = init_crawler(params) #计算每次请求的中心点 print request_points if img_len==0: return "抱歉,任务失败!<br/>框选区域没有雷达降水图。" else: # 初始化文件夹 init_dir(params, request_points) # 保存爬取参数 print "【params】" + str(params) save_json(out_dir, u"0 param - 爬取参数", params) #保存成json file_str = save_params_file(params) #保存给用户看 html_str = file_str.replace('\n', '<br/>') #输出的HTML字符串 # 开启任务,异步进程 executor.submit( start(params, request_points) ) return '任务已在后台运行!<br/>每次需要爬取{}张图片!<br/>{}'.format(img_len, html_str)