def test_schedule(request,id):
#get test
test=TestCase.get(id=id)
#create task chain
tasks=[]
task=Task()
task.target="cimri.module.automation.controller.Controller"
task.op="controller.schedule"
task.threadtags="test: "+test.label
tasks.append(task)
#create test task
task=Task()
task.target=test.module
task.op=test.op
task.meta['test']=True
task.meta['test.uuid']=test.uuid
task.meta['workers']=8
#set task data
task.data=[]
for data in test.data:
taskdata={}
if task.op=="scrap":
taskdata["data"]=data["data"]
taskdata["meta.merchantid"]=data["meta.merchantid"]
if "meta.xmlitem" in data:
taskdata["meta.xmlitem"]={"__class__":"MerchantItem", "dump":MerchantItem(data["meta.xmlitem"]).to_dict()}
if "meta.refitem" in data:
taskdata["meta.refitem"]={"__class__":"MerchantItem", "dump":MerchantItem(data["meta.refitem"]).to_dict()}
else:
taskdata["meta.refitem"]=None
elif task.op=="match":
taskdata["data"]={"__class__":"MerchantItem", "dump":MerchantItem(data["meta.xmlitem"]).to_dict()}
if "meta.refitem" in data:
taskdata["meta.refitem"]={"__class__":"MerchantItem", "dump":MerchantItem(data["meta.refitem"]).to_dict()}
else:
taskdata["meta.refitem"]=None
task.data.append(taskdata)
tasks.append(task)
#schedule
api=ControllerAPI(request)
res=api.schedule(tasks)
#template context
context=_get_context(request,section="common",content="content_test_scheduled")
#render
return HttpResponse( get_template('template_content.html').render(Context(context)) )
def task_schedule(request): #get parameters tags=request.REQUEST["label"] if "label" in request.REQUEST else "" tasks=json.loads(request.REQUEST["tasks"]) if "tasks" in request.REQUEST else [] #validate if len(tasks)==0: return HttpResponse( AIResponse(error=AIError(AIError.SYSTEM)).serialize() ) #create task thread thread=[] task=Task() task.target="cimri.module.automation.controller.Controller" task.op="controller.schedule" thread.append(task) for info in tasks: task=Task() task.target=info["target"] task.op=info["op"] for key in info["meta"]: if info["meta"][key]!="": task.meta[key]=info["meta"][key] thread.append(task) #add label thread[0].threadtags=tags if tags!="" else ", ".join([task.op for task in thread[1:]]) #schedule api=ControllerAPI(request) res=api.schedule(thread) return HttpResponse( AIResponse().serialize() )
def _get_tasks(self,path):
"""
Sistemde kayitli butun Taskleri getirir
@type path: str
@param path: Tasklerin sistemde kayitli oldugu path
@rtype: list (L{cimri.system.task.Task})
@return: istenilen Task'ler
"""
#get task list
files=[ f for f in os.listdir(path) if os.path.isfile( os.path.join(path,f) ) and f.find("-info")>-1 ]
#get tasks
tasks=[]
for file in files:
tasks.append(Task.create_from_file(os.path.join(path,file)))
return tasks
def test_new(request):
#get merchants
api=ControllerAPI(request)
merchants=api.getmerchants()
#create?
if "action" in request.REQUEST and request.REQUEST["action"]=="update":
#get parameters
auto=int(request.REQUEST["auto"])
label=request.REQUEST["label"]
module=request.REQUEST["module"]
merchantid=request.REQUEST["merchant"]
items=request.REQUEST["items"]
#autogenerate samples?
if auto==1:
#find merchant name
mname="*"
if merchantid!="*":
for merchant in merchants:
if merchantid==merchant["id"]:
merchantname=merchant["name"]
#create
data=TestCase()
data.owner=request.session["user"].id
data.label=label
data.module=module
data.op="match" if module.endswith("Matcher") else "scrap"
data.target={"merchant":{"id":merchantid,"name":merchantname}}
data.create()
#create task to generate samples for the training data
tasks=[]
task=Task()
task.target="cimri.module.automation.controller.Controller"
task.op="controller.schedule"
task.threadtags="test: "+label
tasks.append(task)
task=Task()
task.target="cimri.module.crawler.productxml.ProductXMLCrawler"
task.op="sample"
if merchantid!="*":
task.meta["merchants.id"]=int(merchantid)
task.meta["sample.size"]=40
tasks.append(task)
#create samples based on the provided list of item IDs
else:
#get items
input=eval(items.replace("\n",""))
items=[]
for it in input:
item=MerchantItem()
item.merchant={"merchantId":str(it[0]).strip()}
item.merchantItemId=str(it[1]).strip()
item.item={"itemId":str(it[2]).strip()}
items.append(item)
#create
data=TestCase()
data.owner=request.session["user"].id
data.label=label
data.module=module
data.op="match" if module.endswith("Matcher") else "scrap"
data.target={}
data.batch=[item.to_dict() for item in items]
data.create()
#create task to generate samples for the training data
tasks=[]
task=Task()
task.target="cimri.module.automation.controller.Controller"
task.op="controller.schedule"
task.threadtags="test: "+label
tasks.append(task)
task=Task()
task.target="cimri.module.crawler.productxml.ProductXMLCrawler"
task.op="get"
task.data=items
task.meta={"workers":8}
tasks.append(task)
#schedule
api=ControllerAPI(request)
res=api.schedule(tasks)
#get task id
tid=res[1]
#template context
context=_get_context(request,section="common",content="content_test_created",test=data,tid=tid)
else:
#template context
context=_get_context(request,section="common",content="content_test_new",merchants=merchants)
#render
return HttpResponse( get_template('template_content.html').render(Context(context)) )
def training_new(request):
#get merchants
api=ControllerAPI(request)
merchants=api.getmerchants()
#create?
if "action" in request.REQUEST and request.REQUEST["action"]=="update":
#get parameters
label=request.REQUEST["label"]
module=request.REQUEST["module"]
merchantid=request.REQUEST["merchant"]
autotask="system" in request.REQUEST
#find merchant name
mname="*"
if merchantid!="*":
for merchant in merchants:
if merchantid==merchant["id"]:
merchantname=merchant["name"]
#create or update
data=None
if autotask is True:
data=MLCase.get(label=label,module=module)
if data is None:
data=MLCase()
data.owner=request.session["user"].id
data.label=label
data.module=module
data.op="match" if module.endswith("Matcher") else "scrap"
data.target={"merchant":{"id":merchantid,"name":merchantname}}
data.create()
#create task to generate samples for the training data
tasks=[]
task=Task()
task.target="cimri.module.automation.controller.Controller"
task.op="controller.schedule"
task.threadtags="egitim: "+label
tasks.append(task)
task=Task()
task.target="cimri.module.crawler.productxml.ProductXMLCrawler"
task.op="sample"
if merchantid!="*":
task.meta["merchants.id"]=int(merchantid)
task.meta["sample.size"]=40
tasks.append(task)
#schedule
api=ControllerAPI(request)
res=api.schedule(tasks)
#get task id
tid=res[1]
#response
if autotask:
return HttpResponse( AIResponse( data={"id":str(data.id), "tid":tid} ).serialize() )
#template context
context=_get_context(request,section="common",content="content_training_created",data=data,tid=tid)
else:
#template context
context=_get_context(request,section="common",content="content_training_new",merchants=merchants)
#render
return HttpResponse( get_template('template_content.html').render(Context(context)) )
def handle_schedule(request):
#get all templates
templates=TaskTemplate.list()
#get day-of-year, hour and minute
now=datetime.now()
#check for schedule
for template in templates:
#does tempalte have a schedule?
if template.schedule.strip()=="":
continue
#get schedule
times=template.schedule.split(" ")
runtask=False
for t in times:
try:
#get schedule time hour and minute
tokens=t.strip().split(":")
hour=int(tokens[0])
minute=int(tokens[1])
#is it time to schedule this?
if now.hour!=hour or now.minute!=minute:
continue
#get year-month-day
ts=str(now.year)+"."+str(now.month)+"."+str(now.day)
#was this schedule already run today?
if t.strip() in template.schedule_log and template.schedule_log[t.strip()]==ts:
continue
#run rask
runtask=True
#update schedule log
template.schedule_log[t.strip()]=ts
template.save()
#done
break
except:
continue
#run task?
if runtask is False:
continue
#create task thread
thread=[]
task=Task()
task.target="cimri.module.automation.controller.Controller"
task.op="controller.schedule"
thread.append(task)
for info in template.tasks:
task=Task()
task.target=info["target"]
task.op=info["op"]
for key in info["meta"]:
if info["meta"][key]!="":
task.meta[key.replace("-",".")]=info["meta"][key]
thread.append(task)
#add label
# thread[0].threadtags=", ".join([task.op for task in thread[1:]])+" (auto)"
thread[0].threadtags=template.label+" (auto)"
#schedule
api=ControllerAPI(request)
res=api.schedule(thread)
return HttpResponse( AIResponse().serialize() )
"""
def handledata(data,module):
#unpack task
task=Task()
task.unpack(data)
def handleok(data,module,task):
#find thread
thread=None
for t in self.threads:
if t[0]["task"].threadid==task.threadid:
thread=t
break
#check thread
if thread is None:
return #log error (thread not found!)
#find task
index=None
for i in range(len(thread)):
if thread[i]["task"].id==task.id:
index=i
break
#check task
if index is None:
return #log error (task not found!)
#update task
thread[index]["task"]=task
thread[index]["module"]=None
#is there a next task in the thread?
if index==len(thread)-1:
#delete thread, we're done
self.threads.remove(thread)
#append thread to history
self.history.append(thread)
if len(self.history)>10:
self.history=self.history[-10:]
else:
#assign result to next task in thread
thread[index]["task"].load()
thread[index+1]["task"].data=thread[index]["task"].result
thread[index+1]["task"].save()
#free the memory we just used up
thread[index]["task"].unload()
thread[index+1]["task"].unload()
#mark next task on thread as scheduled
thread[index+1]["task"].status=Task.STATUS_SCHEDULED
#done
module.lock=False
module.tasks=module.tasks[1:]
def handleerr(err,module):
#flush call failed. will try again in next monitor cycle
module.lock=False
deferred=module.call("flush")
deferred.addCallback(handleok,module=module,task=task)
deferred.addErrback(handleerr,module=module)
