def start(self):
self._task_fetcher.start()
#self._task_fetcher.join()
self._output_pusher.start()
#self._output_pusher.join()
for worker in self._workers:
worker.start()
def dummy(mission, thread):
"""After download, continue next mission"""
if thread is self.download_thread:
mission = self.get_mission_to_download()
if mission:
worker = self.create_child(download)
worker.start(mission, setting["savepath"])
self.download_thread = worker
else:
safeprint("All download completed")
def handler(event, context, upload_file=True):
_set_env()
tasks = []
for record in event['Records']:
bucket = record['s3']['bucket']['name']
key = record['s3']['object']['key']
key = urllib.unquote(key)
tasks.append((bucket, key))
worker.start(tasks, upload_file)
def start_work(self):
#download
if self.mode == 0:
self.file_data = [[] for i in self.clients]
#upload
else:
self.file_data = fileops.read_file(self.filename,
self.bandwidth_ratio,
self.buff_size)
#Intiate workers
for worker in self.workers:
worker.start()
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from configmgr import configmgr
from singleton import singleton
import worker
import testroutine
@singleton
class sometestcls:
def __init__(self):
pass
if __name__ == "__main__":
configmgr().read('./config.json')
testroutine.run()
worker.start()
'''
Created on 2013-2-16
@author: Administrator
'''
import worker
import multiprocessing
class SomeObject():
def __init__(self,name,sex):
self._name=name
self._sex=sex
def say(self):
print('%s is %s old' %(self._name,self._sex))
if __name__=='__main__':
obj=SomeObject('zhang','30')
queue=multiprocessing.Queue()
worker=worker.Worker(queue)
worker.start()
queue.put(obj)
print ('in main process %d' %(multiprocessing.current_process().pid))
worker.join()
def initparam():
""" 初始化参数
"""
params = OrderedDict()
print "【从前端获得参数】 %s" % str(request.values)
print "【正在处理参数】"
# 【开始时间】
params["start_time"] = request.form.get("startTime", type=str)
if is_timestr(params['start_time']) is False:
params['start_time'] = get_time()
print "\t[开始时间] %s" % params["start_time"]
# 【结束时间】
params["end_time"] = request.form.get("endTime", type=str)
if is_timestr(params['end_time']) is False:
params['end_time'] = "未设置" # 不结束
print "\t[结束时间] %s" %params["end_time"]
# 【时间间隔】
params["interval"] = request.form.get("interval", type=int)
params['step'] = params['interval'] / 5 # 步长
print "\t[时间间隔] %d" % params["interval"]
print "\t[保存图片的步长] %d" % params["step"]
# 【处理点】
points = {
'center_point' : request.form.get("centerPoint", type=str),
'north_west_point' : request.form.get("northWestPoint", type=str),
'north_east_point' : request.form.get("northEastPoint", type=str),
'south_east_point' : request.form.get("southEastPoint", type=str),
'south_west_point' : request.form.get("southWestPoint", type=str)
}
print '\t[点] %s' % str(points)
for point_name,value in points.items():
params[point_name] = get_latlng(
value
)
# 【保存文件夹】
params["save_file_dir"] = request.form.get("saveFileDir", type=str)
print "\t[保存文件夹] %s" % params["save_file_dir"]
save_dir = params["save_file_dir"]
if not os.path.exists(save_dir):
# 选择的文件夹不存在
print "\t【WARNING】 选择的文件夹不存在，跳转到初始页面"
print "【params】" + str(params)
return redirect_index("文件夹路径不存在，请重新输入！")
# # 【输出文件夹】
out_dir = os.path.join(save_dir, params['start_time'])
params['out_dir'] = out_dir
print "\t[图像输出文件夹] %s" % params["out_dir"]
if os.path.exists(out_dir):
# 输出文件夹已经存在
print "【params】" + str(params)
return redirect_index("输出文件夹已经存在，%s<Br/>请重新输入文件夹！" % out_dir)
os.makedirs(out_dir)
# 项目备注
params["remark"] = request.form.get("remark")
print "\t[项目备注] %s" % params["remark"]
img_len , request_points = init_crawler(params) #计算每次请求的中心点
print request_points
if img_len==0:
return "抱歉，任务失败！<br/>框选区域没有雷达降水图。"
else:
# 初始化文件夹
init_dir(params, request_points)
# 保存爬取参数
print "【params】" + str(params)
save_json(out_dir, u"0 param - 爬取参数", params) #保存成json
file_str = save_params_file(params) #保存给用户看
html_str = file_str.replace('\n', '<br/>') #输出的HTML字符串
# 开启任务，异步进程
executor.submit(
start(params, request_points)
)
return '任务已在后台运行！<br/>每次需要爬取{}张图片！<br/>{}'.format(img_len, html_str)
