Exemple #1
0
def task_schedule(request):
	#get parameters
	tags=request.REQUEST["label"] if "label" in request.REQUEST else ""
	tasks=json.loads(request.REQUEST["tasks"]) if "tasks" in request.REQUEST else []

	#validate
	if len(tasks)==0:
		return HttpResponse( AIResponse(error=AIError(AIError.SYSTEM)).serialize() )

	#create task thread
	thread=[]
	task=Task()
	task.target="cimri.module.automation.controller.Controller"
	task.op="controller.schedule"
	thread.append(task)
	for info in tasks:
	        task=Task()
		task.target=info["target"]
		task.op=info["op"]
		for key in info["meta"]:
			if info["meta"][key]!="":
			        task.meta[key]=info["meta"][key]             
		thread.append(task)

	#add label
	thread[0].threadtags=tags if tags!="" else ", ".join([task.op for task in thread[1:]])

	#schedule
	api=ControllerAPI(request)
	res=api.schedule(thread)

	return HttpResponse( AIResponse().serialize() )
Exemple #2
0
def test_schedule(request,id):
	#get test
	test=TestCase.get(id=id)

	#create task chain
	tasks=[]
	task=Task()
	task.target="cimri.module.automation.controller.Controller"
	task.op="controller.schedule"
	task.threadtags="test: "+test.label
	tasks.append(task)

	#create test task
	task=Task()
	task.target=test.module
	task.op=test.op
	task.meta['test']=True
	task.meta['test.uuid']=test.uuid
	task.meta['workers']=8

	#set task data
	task.data=[]
	for data in test.data:
		taskdata={}
		
		if task.op=="scrap":
			taskdata["data"]=data["data"]
			taskdata["meta.merchantid"]=data["meta.merchantid"]
			if "meta.xmlitem" in data:
				taskdata["meta.xmlitem"]={"__class__":"MerchantItem", "dump":MerchantItem(data["meta.xmlitem"]).to_dict()}
			if "meta.refitem" in data:
				taskdata["meta.refitem"]={"__class__":"MerchantItem", "dump":MerchantItem(data["meta.refitem"]).to_dict()}
			else:
				taskdata["meta.refitem"]=None

		elif task.op=="match":		
			taskdata["data"]={"__class__":"MerchantItem", "dump":MerchantItem(data["meta.xmlitem"]).to_dict()}
			if "meta.refitem" in data:
				taskdata["meta.refitem"]={"__class__":"MerchantItem", "dump":MerchantItem(data["meta.refitem"]).to_dict()}
			else:
				taskdata["meta.refitem"]=None

		task.data.append(taskdata)

	tasks.append(task)
	
	
	#schedule
	api=ControllerAPI(request)
	res=api.schedule(tasks)
		        
	#template context
	context=_get_context(request,section="common",content="content_test_scheduled")

	#render
        return HttpResponse( get_template('template_content.html').render(Context(context)) )
Exemple #3
0
def test_new(request):
	#get merchants
	api=ControllerAPI(request)
	merchants=api.getmerchants()

	#create?
	if "action" in request.REQUEST and request.REQUEST["action"]=="update":
		#get parameters
		auto=int(request.REQUEST["auto"])
		label=request.REQUEST["label"]
		module=request.REQUEST["module"]
		merchantid=request.REQUEST["merchant"]
		items=request.REQUEST["items"]
		
		#autogenerate samples?
		if auto==1:		
			#find merchant name
			mname="*"
			if merchantid!="*":
				for merchant in merchants:
					if merchantid==merchant["id"]:
						merchantname=merchant["name"]
		
			#create
			data=TestCase()
			data.owner=request.session["user"].id
			data.label=label
			data.module=module
			data.op="match" if module.endswith("Matcher") else "scrap"
			data.target={"merchant":{"id":merchantid,"name":merchantname}}
			data.create()

			#create task to generate samples for the training data
			tasks=[]
			task=Task()
			task.target="cimri.module.automation.controller.Controller"
			task.op="controller.schedule"
			task.threadtags="test: "+label
			tasks.append(task)
       		 	task=Task()
 	      	 	task.target="cimri.module.crawler.productxml.ProductXMLCrawler"
 			task.op="sample"
			if merchantid!="*":
				task.meta["merchants.id"]=int(merchantid)
			task.meta["sample.size"]=40     
			tasks.append(task)

		#create samples based on the provided list of item IDs
		else:
			#get items
			input=eval(items.replace("\n",""))
			items=[]
			for it in input:
				item=MerchantItem()
				item.merchant={"merchantId":str(it[0]).strip()}
				item.merchantItemId=str(it[1]).strip()
				item.item={"itemId":str(it[2]).strip()}
				items.append(item)				

			#create
			data=TestCase()
			data.owner=request.session["user"].id
			data.label=label
			data.module=module
			data.op="match" if module.endswith("Matcher") else "scrap"
			data.target={}
			data.batch=[item.to_dict() for item in items]
			data.create()

			#create task to generate samples for the training data
			tasks=[]
			task=Task()
			task.target="cimri.module.automation.controller.Controller"
			task.op="controller.schedule"
			task.threadtags="test: "+label
			tasks.append(task)
       		 	task=Task()
 	      	 	task.target="cimri.module.crawler.productxml.ProductXMLCrawler"
 			task.op="get"
			task.data=items
			task.meta={"workers":8}
			tasks.append(task)

		#schedule
		api=ControllerAPI(request)
		res=api.schedule(tasks)

		#get task id			
		tid=res[1]

	        #template context
		context=_get_context(request,section="common",content="content_test_created",test=data,tid=tid)

	else:
	        #template context
		context=_get_context(request,section="common",content="content_test_new",merchants=merchants)

	#render
        return HttpResponse( get_template('template_content.html').render(Context(context)) )
Exemple #4
0
def training_new(request):
	#get merchants
	api=ControllerAPI(request)
	merchants=api.getmerchants()

	#create?
	if "action" in request.REQUEST and request.REQUEST["action"]=="update":
		#get parameters
		label=request.REQUEST["label"]
		module=request.REQUEST["module"]
		merchantid=request.REQUEST["merchant"]
		autotask="system" in request.REQUEST
		
		#find merchant name
		mname="*"
		if merchantid!="*":
			for merchant in merchants:
				if merchantid==merchant["id"]:
					merchantname=merchant["name"]
		
		#create or update
		data=None
		if autotask is True:
			data=MLCase.get(label=label,module=module)
		if data is None:
			data=MLCase()
		data.owner=request.session["user"].id
		data.label=label
		data.module=module
		data.op="match" if module.endswith("Matcher") else "scrap"
		data.target={"merchant":{"id":merchantid,"name":merchantname}}
		data.create()

		#create task to generate samples for the training data
		tasks=[]
		task=Task()
		task.target="cimri.module.automation.controller.Controller"
		task.op="controller.schedule"
		task.threadtags="egitim: "+label
		tasks.append(task)
       	 	task=Task()
        	task.target="cimri.module.crawler.productxml.ProductXMLCrawler"
 		task.op="sample"
		if merchantid!="*":
			task.meta["merchants.id"]=int(merchantid)
		task.meta["sample.size"]=40     
		tasks.append(task)

		#schedule
		api=ControllerAPI(request)
		res=api.schedule(tasks)

		#get task id 
		tid=res[1]

		#response
		if autotask:
			return HttpResponse( AIResponse( data={"id":str(data.id), "tid":tid} ).serialize() )

	        #template context
		context=_get_context(request,section="common",content="content_training_created",data=data,tid=tid)

	else:
	        #template context
		context=_get_context(request,section="common",content="content_training_new",merchants=merchants)

	#render
        return HttpResponse( get_template('template_content.html').render(Context(context)) )
Exemple #5
0
def handle_schedule(request):
	#get all templates
	templates=TaskTemplate.list()

	#get day-of-year, hour and minute
	now=datetime.now()

	#check for schedule
	for template in templates:
		#does tempalte have a schedule?
		if template.schedule.strip()=="":
			continue

		#get schedule
		times=template.schedule.split(" ")
		runtask=False
		for t in times:
			try:
				#get schedule time hour and minute
				tokens=t.strip().split(":")
				hour=int(tokens[0])
				minute=int(tokens[1])

				#is it time to schedule this?
				if now.hour!=hour or now.minute!=minute:
					continue

				#get year-month-day
				ts=str(now.year)+"."+str(now.month)+"."+str(now.day)				

				#was this schedule already run today?
				if t.strip() in template.schedule_log and template.schedule_log[t.strip()]==ts:
					continue

				#run rask
				runtask=True

				#update schedule log
				template.schedule_log[t.strip()]=ts
				template.save()

				#done
				break

			except:
				continue
		
		#run task?
		if runtask is False:
			continue

		#create task thread
		thread=[]
		task=Task()
		task.target="cimri.module.automation.controller.Controller"
		task.op="controller.schedule"
		thread.append(task)
		for info in template.tasks:
		        task=Task()
			task.target=info["target"]
			task.op=info["op"]

			for key in info["meta"]:
				if info["meta"][key]!="":
				        task.meta[key.replace("-",".")]=info["meta"][key]  
			thread.append(task)

		#add label
#		thread[0].threadtags=", ".join([task.op for task in thread[1:]])+" (auto)"
		thread[0].threadtags=template.label+" (auto)"

		#schedule
		api=ControllerAPI(request)
		res=api.schedule(thread)
		

	return HttpResponse( AIResponse().serialize() )