def task_schedule(request): #get parameters tags=request.REQUEST["label"] if "label" in request.REQUEST else "" tasks=json.loads(request.REQUEST["tasks"]) if "tasks" in request.REQUEST else [] #validate if len(tasks)==0: return HttpResponse( AIResponse(error=AIError(AIError.SYSTEM)).serialize() ) #create task thread thread=[] task=Task() task.target="cimri.module.automation.controller.Controller" task.op="controller.schedule" thread.append(task) for info in tasks: task=Task() task.target=info["target"] task.op=info["op"] for key in info["meta"]: if info["meta"][key]!="": task.meta[key]=info["meta"][key] thread.append(task) #add label thread[0].threadtags=tags if tags!="" else ", ".join([task.op for task in thread[1:]]) #schedule api=ControllerAPI(request) res=api.schedule(thread) return HttpResponse( AIResponse().serialize() )
def test_schedule(request,id): #get test test=TestCase.get(id=id) #create task chain tasks=[] task=Task() task.target="cimri.module.automation.controller.Controller" task.op="controller.schedule" task.threadtags="test: "+test.label tasks.append(task) #create test task task=Task() task.target=test.module task.op=test.op task.meta['test']=True task.meta['test.uuid']=test.uuid task.meta['workers']=8 #set task data task.data=[] for data in test.data: taskdata={} if task.op=="scrap": taskdata["data"]=data["data"] taskdata["meta.merchantid"]=data["meta.merchantid"] if "meta.xmlitem" in data: taskdata["meta.xmlitem"]={"__class__":"MerchantItem", "dump":MerchantItem(data["meta.xmlitem"]).to_dict()} if "meta.refitem" in data: taskdata["meta.refitem"]={"__class__":"MerchantItem", "dump":MerchantItem(data["meta.refitem"]).to_dict()} else: taskdata["meta.refitem"]=None elif task.op=="match": taskdata["data"]={"__class__":"MerchantItem", "dump":MerchantItem(data["meta.xmlitem"]).to_dict()} if "meta.refitem" in data: taskdata["meta.refitem"]={"__class__":"MerchantItem", "dump":MerchantItem(data["meta.refitem"]).to_dict()} else: taskdata["meta.refitem"]=None task.data.append(taskdata) tasks.append(task) #schedule api=ControllerAPI(request) res=api.schedule(tasks) #template context context=_get_context(request,section="common",content="content_test_scheduled") #render return HttpResponse( get_template('template_content.html').render(Context(context)) )
def test_new(request): #get merchants api=ControllerAPI(request) merchants=api.getmerchants() #create? if "action" in request.REQUEST and request.REQUEST["action"]=="update": #get parameters auto=int(request.REQUEST["auto"]) label=request.REQUEST["label"] module=request.REQUEST["module"] merchantid=request.REQUEST["merchant"] items=request.REQUEST["items"] #autogenerate samples? if auto==1: #find merchant name mname="*" if merchantid!="*": for merchant in merchants: if merchantid==merchant["id"]: merchantname=merchant["name"] #create data=TestCase() data.owner=request.session["user"].id data.label=label data.module=module data.op="match" if module.endswith("Matcher") else "scrap" data.target={"merchant":{"id":merchantid,"name":merchantname}} data.create() #create task to generate samples for the training data tasks=[] task=Task() task.target="cimri.module.automation.controller.Controller" task.op="controller.schedule" task.threadtags="test: "+label tasks.append(task) task=Task() task.target="cimri.module.crawler.productxml.ProductXMLCrawler" task.op="sample" if merchantid!="*": task.meta["merchants.id"]=int(merchantid) task.meta["sample.size"]=40 tasks.append(task) #create samples based on the provided list of item IDs else: #get items input=eval(items.replace("\n","")) items=[] for it in input: item=MerchantItem() item.merchant={"merchantId":str(it[0]).strip()} item.merchantItemId=str(it[1]).strip() item.item={"itemId":str(it[2]).strip()} items.append(item) #create data=TestCase() data.owner=request.session["user"].id data.label=label data.module=module data.op="match" if module.endswith("Matcher") else "scrap" data.target={} data.batch=[item.to_dict() for item in items] data.create() #create task to generate samples for the training data tasks=[] task=Task() task.target="cimri.module.automation.controller.Controller" task.op="controller.schedule" task.threadtags="test: "+label tasks.append(task) task=Task() task.target="cimri.module.crawler.productxml.ProductXMLCrawler" task.op="get" task.data=items task.meta={"workers":8} tasks.append(task) #schedule api=ControllerAPI(request) res=api.schedule(tasks) #get task id tid=res[1] #template context context=_get_context(request,section="common",content="content_test_created",test=data,tid=tid) else: #template context context=_get_context(request,section="common",content="content_test_new",merchants=merchants) #render return HttpResponse( get_template('template_content.html').render(Context(context)) )
def training_new(request): #get merchants api=ControllerAPI(request) merchants=api.getmerchants() #create? if "action" in request.REQUEST and request.REQUEST["action"]=="update": #get parameters label=request.REQUEST["label"] module=request.REQUEST["module"] merchantid=request.REQUEST["merchant"] autotask="system" in request.REQUEST #find merchant name mname="*" if merchantid!="*": for merchant in merchants: if merchantid==merchant["id"]: merchantname=merchant["name"] #create or update data=None if autotask is True: data=MLCase.get(label=label,module=module) if data is None: data=MLCase() data.owner=request.session["user"].id data.label=label data.module=module data.op="match" if module.endswith("Matcher") else "scrap" data.target={"merchant":{"id":merchantid,"name":merchantname}} data.create() #create task to generate samples for the training data tasks=[] task=Task() task.target="cimri.module.automation.controller.Controller" task.op="controller.schedule" task.threadtags="egitim: "+label tasks.append(task) task=Task() task.target="cimri.module.crawler.productxml.ProductXMLCrawler" task.op="sample" if merchantid!="*": task.meta["merchants.id"]=int(merchantid) task.meta["sample.size"]=40 tasks.append(task) #schedule api=ControllerAPI(request) res=api.schedule(tasks) #get task id tid=res[1] #response if autotask: return HttpResponse( AIResponse( data={"id":str(data.id), "tid":tid} ).serialize() ) #template context context=_get_context(request,section="common",content="content_training_created",data=data,tid=tid) else: #template context context=_get_context(request,section="common",content="content_training_new",merchants=merchants) #render return HttpResponse( get_template('template_content.html').render(Context(context)) )
def handle_schedule(request): #get all templates templates=TaskTemplate.list() #get day-of-year, hour and minute now=datetime.now() #check for schedule for template in templates: #does tempalte have a schedule? if template.schedule.strip()=="": continue #get schedule times=template.schedule.split(" ") runtask=False for t in times: try: #get schedule time hour and minute tokens=t.strip().split(":") hour=int(tokens[0]) minute=int(tokens[1]) #is it time to schedule this? if now.hour!=hour or now.minute!=minute: continue #get year-month-day ts=str(now.year)+"."+str(now.month)+"."+str(now.day) #was this schedule already run today? if t.strip() in template.schedule_log and template.schedule_log[t.strip()]==ts: continue #run rask runtask=True #update schedule log template.schedule_log[t.strip()]=ts template.save() #done break except: continue #run task? if runtask is False: continue #create task thread thread=[] task=Task() task.target="cimri.module.automation.controller.Controller" task.op="controller.schedule" thread.append(task) for info in template.tasks: task=Task() task.target=info["target"] task.op=info["op"] for key in info["meta"]: if info["meta"][key]!="": task.meta[key.replace("-",".")]=info["meta"][key] thread.append(task) #add label # thread[0].threadtags=", ".join([task.op for task in thread[1:]])+" (auto)" thread[0].threadtags=template.label+" (auto)" #schedule api=ControllerAPI(request) res=api.schedule(thread) return HttpResponse( AIResponse().serialize() )