def scraper_reports(request):
#get merchants
api=ControllerAPI(request)
merchants=api.getmerchants()
if merchants is None:
merchants=[]
#get scraper status for each merchant
api=ScraperAPI(request)
statuses=api.getscraperstatus()
#create status for merchants that do not have a stored status
ids=[str(status.merchantid) for status in statuses]
for merchant in merchants:
if str(merchant["id"]) not in ids:
s=api.createmerchant(merchant["id"])
#convert merchant ids to ints
for merchant in merchants:
merchant["id"]=int(merchant["id"])
#get statuses again
statuses=api.getscraperstatus()
#template context
context=_get_context(request,section="admin",content="content_scraper_reports",merchants=merchants,statuses=statuses)
#render
return HttpResponse( get_template('template_cpanel.html').render(Context(context)) )
def task_create(request):
#get descriptors and merchants
api=ControllerAPI(request)
descriptors=api.getmoduledescriptors()
merchants=api.getmerchants()
#template context
context=_get_context(request,content="content_task_addedit",descriptors=json.dumps(descriptors),merchants=json.dumps(merchants))
#render
return HttpResponse( get_template('template_content.html').render(Context(context)) )
def create_task_from_template(request,id):
#get template
template=TaskTemplate.get(id=id)
#get descriptors and merchants
api=ControllerAPI(request)
descriptors=api.getmoduledescriptors()
merchants=api.getmerchants()
#template context
context=_get_context(request,content="content_task_addedit",descriptors=json.dumps(descriptors),merchants=json.dumps(merchants),template=template,tasks=json.dumps(template.gettasks()))
#render
return HttpResponse( get_template('template_content.html').render(Context(context)) )
def test_new(request):
#get merchants
api=ControllerAPI(request)
merchants=api.getmerchants()
#create?
if "action" in request.REQUEST and request.REQUEST["action"]=="update":
#get parameters
auto=int(request.REQUEST["auto"])
label=request.REQUEST["label"]
module=request.REQUEST["module"]
merchantid=request.REQUEST["merchant"]
items=request.REQUEST["items"]
#autogenerate samples?
if auto==1:
#find merchant name
mname="*"
if merchantid!="*":
for merchant in merchants:
if merchantid==merchant["id"]:
merchantname=merchant["name"]
#create
data=TestCase()
data.owner=request.session["user"].id
data.label=label
data.module=module
data.op="match" if module.endswith("Matcher") else "scrap"
data.target={"merchant":{"id":merchantid,"name":merchantname}}
data.create()
#create task to generate samples for the training data
tasks=[]
task=Task()
task.target="cimri.module.automation.controller.Controller"
task.op="controller.schedule"
task.threadtags="test: "+label
tasks.append(task)
task=Task()
task.target="cimri.module.crawler.productxml.ProductXMLCrawler"
task.op="sample"
if merchantid!="*":
task.meta["merchants.id"]=int(merchantid)
task.meta["sample.size"]=40
tasks.append(task)
#create samples based on the provided list of item IDs
else:
#get items
input=eval(items.replace("\n",""))
items=[]
for it in input:
item=MerchantItem()
item.merchant={"merchantId":str(it[0]).strip()}
item.merchantItemId=str(it[1]).strip()
item.item={"itemId":str(it[2]).strip()}
items.append(item)
#create
data=TestCase()
data.owner=request.session["user"].id
data.label=label
data.module=module
data.op="match" if module.endswith("Matcher") else "scrap"
data.target={}
data.batch=[item.to_dict() for item in items]
data.create()
#create task to generate samples for the training data
tasks=[]
task=Task()
task.target="cimri.module.automation.controller.Controller"
task.op="controller.schedule"
task.threadtags="test: "+label
tasks.append(task)
task=Task()
task.target="cimri.module.crawler.productxml.ProductXMLCrawler"
task.op="get"
task.data=items
task.meta={"workers":8}
tasks.append(task)
#schedule
api=ControllerAPI(request)
res=api.schedule(tasks)
#get task id
tid=res[1]
#template context
context=_get_context(request,section="common",content="content_test_created",test=data,tid=tid)
else:
#template context
context=_get_context(request,section="common",content="content_test_new",merchants=merchants)
#render
return HttpResponse( get_template('template_content.html').render(Context(context)) )
def training_new(request):
#get merchants
api=ControllerAPI(request)
merchants=api.getmerchants()
#create?
if "action" in request.REQUEST and request.REQUEST["action"]=="update":
#get parameters
label=request.REQUEST["label"]
module=request.REQUEST["module"]
merchantid=request.REQUEST["merchant"]
autotask="system" in request.REQUEST
#find merchant name
mname="*"
if merchantid!="*":
for merchant in merchants:
if merchantid==merchant["id"]:
merchantname=merchant["name"]
#create or update
data=None
if autotask is True:
data=MLCase.get(label=label,module=module)
if data is None:
data=MLCase()
data.owner=request.session["user"].id
data.label=label
data.module=module
data.op="match" if module.endswith("Matcher") else "scrap"
data.target={"merchant":{"id":merchantid,"name":merchantname}}
data.create()
#create task to generate samples for the training data
tasks=[]
task=Task()
task.target="cimri.module.automation.controller.Controller"
task.op="controller.schedule"
task.threadtags="egitim: "+label
tasks.append(task)
task=Task()
task.target="cimri.module.crawler.productxml.ProductXMLCrawler"
task.op="sample"
if merchantid!="*":
task.meta["merchants.id"]=int(merchantid)
task.meta["sample.size"]=40
tasks.append(task)
#schedule
api=ControllerAPI(request)
res=api.schedule(tasks)
#get task id
tid=res[1]
#response
if autotask:
return HttpResponse( AIResponse( data={"id":str(data.id), "tid":tid} ).serialize() )
#template context
context=_get_context(request,section="common",content="content_training_created",data=data,tid=tid)
else:
#template context
context=_get_context(request,section="common",content="content_training_new",merchants=merchants)
#render
return HttpResponse( get_template('template_content.html').render(Context(context)) )
