def test_schedule(request,id): #get test test=TestCase.get(id=id) #create task chain tasks=[] task=Task() task.target="cimri.module.automation.controller.Controller" task.op="controller.schedule" task.threadtags="test: "+test.label tasks.append(task) #create test task task=Task() task.target=test.module task.op=test.op task.meta['test']=True task.meta['test.uuid']=test.uuid task.meta['workers']=8 #set task data task.data=[] for data in test.data: taskdata={} if task.op=="scrap": taskdata["data"]=data["data"] taskdata["meta.merchantid"]=data["meta.merchantid"] if "meta.xmlitem" in data: taskdata["meta.xmlitem"]={"__class__":"MerchantItem", "dump":MerchantItem(data["meta.xmlitem"]).to_dict()} if "meta.refitem" in data: taskdata["meta.refitem"]={"__class__":"MerchantItem", "dump":MerchantItem(data["meta.refitem"]).to_dict()} else: taskdata["meta.refitem"]=None elif task.op=="match": taskdata["data"]={"__class__":"MerchantItem", "dump":MerchantItem(data["meta.xmlitem"]).to_dict()} if "meta.refitem" in data: taskdata["meta.refitem"]={"__class__":"MerchantItem", "dump":MerchantItem(data["meta.refitem"]).to_dict()} else: taskdata["meta.refitem"]=None task.data.append(taskdata) tasks.append(task) #schedule api=ControllerAPI(request) res=api.schedule(tasks) #template context context=_get_context(request,section="common",content="content_test_scheduled") #render return HttpResponse( get_template('template_content.html').render(Context(context)) )
def task_schedule(request): #get parameters tags=request.REQUEST["label"] if "label" in request.REQUEST else "" tasks=json.loads(request.REQUEST["tasks"]) if "tasks" in request.REQUEST else [] #validate if len(tasks)==0: return HttpResponse( AIResponse(error=AIError(AIError.SYSTEM)).serialize() ) #create task thread thread=[] task=Task() task.target="cimri.module.automation.controller.Controller" task.op="controller.schedule" thread.append(task) for info in tasks: task=Task() task.target=info["target"] task.op=info["op"] for key in info["meta"]: if info["meta"][key]!="": task.meta[key]=info["meta"][key] thread.append(task) #add label thread[0].threadtags=tags if tags!="" else ", ".join([task.op for task in thread[1:]]) #schedule api=ControllerAPI(request) res=api.schedule(thread) return HttpResponse( AIResponse().serialize() )
def _get_tasks(self,path): """ Sistemde kayitli butun Taskleri getirir @type path: str @param path: Tasklerin sistemde kayitli oldugu path @rtype: list (L{cimri.system.task.Task}) @return: istenilen Task'ler """ #get task list files=[ f for f in os.listdir(path) if os.path.isfile( os.path.join(path,f) ) and f.find("-info")>-1 ] #get tasks tasks=[] for file in files: tasks.append(Task.create_from_file(os.path.join(path,file))) return tasks
def test_new(request): #get merchants api=ControllerAPI(request) merchants=api.getmerchants() #create? if "action" in request.REQUEST and request.REQUEST["action"]=="update": #get parameters auto=int(request.REQUEST["auto"]) label=request.REQUEST["label"] module=request.REQUEST["module"] merchantid=request.REQUEST["merchant"] items=request.REQUEST["items"] #autogenerate samples? if auto==1: #find merchant name mname="*" if merchantid!="*": for merchant in merchants: if merchantid==merchant["id"]: merchantname=merchant["name"] #create data=TestCase() data.owner=request.session["user"].id data.label=label data.module=module data.op="match" if module.endswith("Matcher") else "scrap" data.target={"merchant":{"id":merchantid,"name":merchantname}} data.create() #create task to generate samples for the training data tasks=[] task=Task() task.target="cimri.module.automation.controller.Controller" task.op="controller.schedule" task.threadtags="test: "+label tasks.append(task) task=Task() task.target="cimri.module.crawler.productxml.ProductXMLCrawler" task.op="sample" if merchantid!="*": task.meta["merchants.id"]=int(merchantid) task.meta["sample.size"]=40 tasks.append(task) #create samples based on the provided list of item IDs else: #get items input=eval(items.replace("\n","")) items=[] for it in input: item=MerchantItem() item.merchant={"merchantId":str(it[0]).strip()} item.merchantItemId=str(it[1]).strip() item.item={"itemId":str(it[2]).strip()} items.append(item) #create data=TestCase() data.owner=request.session["user"].id data.label=label data.module=module data.op="match" if module.endswith("Matcher") else "scrap" data.target={} data.batch=[item.to_dict() for item in items] data.create() #create task to generate samples for the training data tasks=[] task=Task() task.target="cimri.module.automation.controller.Controller" task.op="controller.schedule" task.threadtags="test: "+label tasks.append(task) task=Task() task.target="cimri.module.crawler.productxml.ProductXMLCrawler" task.op="get" task.data=items task.meta={"workers":8} tasks.append(task) #schedule api=ControllerAPI(request) res=api.schedule(tasks) #get task id tid=res[1] #template context context=_get_context(request,section="common",content="content_test_created",test=data,tid=tid) else: #template context context=_get_context(request,section="common",content="content_test_new",merchants=merchants) #render return HttpResponse( get_template('template_content.html').render(Context(context)) )
def training_new(request): #get merchants api=ControllerAPI(request) merchants=api.getmerchants() #create? if "action" in request.REQUEST and request.REQUEST["action"]=="update": #get parameters label=request.REQUEST["label"] module=request.REQUEST["module"] merchantid=request.REQUEST["merchant"] autotask="system" in request.REQUEST #find merchant name mname="*" if merchantid!="*": for merchant in merchants: if merchantid==merchant["id"]: merchantname=merchant["name"] #create or update data=None if autotask is True: data=MLCase.get(label=label,module=module) if data is None: data=MLCase() data.owner=request.session["user"].id data.label=label data.module=module data.op="match" if module.endswith("Matcher") else "scrap" data.target={"merchant":{"id":merchantid,"name":merchantname}} data.create() #create task to generate samples for the training data tasks=[] task=Task() task.target="cimri.module.automation.controller.Controller" task.op="controller.schedule" task.threadtags="egitim: "+label tasks.append(task) task=Task() task.target="cimri.module.crawler.productxml.ProductXMLCrawler" task.op="sample" if merchantid!="*": task.meta["merchants.id"]=int(merchantid) task.meta["sample.size"]=40 tasks.append(task) #schedule api=ControllerAPI(request) res=api.schedule(tasks) #get task id tid=res[1] #response if autotask: return HttpResponse( AIResponse( data={"id":str(data.id), "tid":tid} ).serialize() ) #template context context=_get_context(request,section="common",content="content_training_created",data=data,tid=tid) else: #template context context=_get_context(request,section="common",content="content_training_new",merchants=merchants) #render return HttpResponse( get_template('template_content.html').render(Context(context)) )
def handle_schedule(request): #get all templates templates=TaskTemplate.list() #get day-of-year, hour and minute now=datetime.now() #check for schedule for template in templates: #does tempalte have a schedule? if template.schedule.strip()=="": continue #get schedule times=template.schedule.split(" ") runtask=False for t in times: try: #get schedule time hour and minute tokens=t.strip().split(":") hour=int(tokens[0]) minute=int(tokens[1]) #is it time to schedule this? if now.hour!=hour or now.minute!=minute: continue #get year-month-day ts=str(now.year)+"."+str(now.month)+"."+str(now.day) #was this schedule already run today? if t.strip() in template.schedule_log and template.schedule_log[t.strip()]==ts: continue #run rask runtask=True #update schedule log template.schedule_log[t.strip()]=ts template.save() #done break except: continue #run task? if runtask is False: continue #create task thread thread=[] task=Task() task.target="cimri.module.automation.controller.Controller" task.op="controller.schedule" thread.append(task) for info in template.tasks: task=Task() task.target=info["target"] task.op=info["op"] for key in info["meta"]: if info["meta"][key]!="": task.meta[key.replace("-",".")]=info["meta"][key] thread.append(task) #add label # thread[0].threadtags=", ".join([task.op for task in thread[1:]])+" (auto)" thread[0].threadtags=template.label+" (auto)" #schedule api=ControllerAPI(request) res=api.schedule(thread) return HttpResponse( AIResponse().serialize() )
"""
def handledata(data,module): #unpack task task=Task() task.unpack(data) def handleok(data,module,task): #find thread thread=None for t in self.threads: if t[0]["task"].threadid==task.threadid: thread=t break #check thread if thread is None: return #log error (thread not found!) #find task index=None for i in range(len(thread)): if thread[i]["task"].id==task.id: index=i break #check task if index is None: return #log error (task not found!) #update task thread[index]["task"]=task thread[index]["module"]=None #is there a next task in the thread? if index==len(thread)-1: #delete thread, we're done self.threads.remove(thread) #append thread to history self.history.append(thread) if len(self.history)>10: self.history=self.history[-10:] else: #assign result to next task in thread thread[index]["task"].load() thread[index+1]["task"].data=thread[index]["task"].result thread[index+1]["task"].save() #free the memory we just used up thread[index]["task"].unload() thread[index+1]["task"].unload() #mark next task on thread as scheduled thread[index+1]["task"].status=Task.STATUS_SCHEDULED #done module.lock=False module.tasks=module.tasks[1:] def handleerr(err,module): #flush call failed. will try again in next monitor cycle module.lock=False deferred=module.call("flush") deferred.addCallback(handleok,module=module,task=task) deferred.addErrback(handleerr,module=module)