def import_all(cls):
cls.objects.all().delete()
prm = create_prm(cls.RDF_TYPE)
res = request_metro(prm)
data = parse_response(res)
instances = []
for d in data:
try:
date = dp.parse(d['dc:date'])
conn_railways = ','.join(d['odpt:connectingRailway'])
exits = ','.join(d['odpt:exit']) if d['odpt:exit'] else ''
surveys = ','.join(d['odpt:passengerSurvey'])
instances.append(
cls(ld_context=d['@context'],
ld_id=d['@id'],
ld_type=d['@type'],
same_as=d['owl:sameAs'],
date=date,
title=d['dc:title'],
connecting_railways=conn_railways,
exits=exits,
facility=d['odpt:facility'],
operator=d['odpt:operator'],
passenger_surveys=surveys,
railway_id=d['odpt:railway'],
station_code=d['odpt:stationCode'],
region=d['ug:region']))
except:
print d
raise
for ins in instances:
ins.save()
def import_all(cls):
cls.objects.all().delete()
prm = create_prm(cls.RDF_TYPE)
res = request_metro(prm)
data = parse_response(res)
for d in data:
try:
date = dp.parse(d['dc:date'])
rw = cls(ld_context=d['@context'],
ld_id=d['@id'],
ld_type=d['@type'],
same_as=d['owl:sameAs'],
date=date,
title=d['dc:title'],
operator=d['odpt:operator'],
line_code=d['odpt:lineCode'],
region=d['ug:region'])
rw.save()
# save StationOrder
so_list = d['odpt:stationOrder']
for so in so_list:
st = Station.get_or_create(so['odpt:station'])
StationOrder.get_or_create(rw, st, so['odpt:index'])
# save TravelTime
tt_list = d['odpt:travelTime']
for tt in tt_list:
fs = Station.get_or_create(tt['odpt:fromStation'])
ts = Station.get_or_create(tt['odpt:toStation'])
TravelTime.get_or_create(rw, fs, ts,
tt['odpt:necessaryTime'],
tt['odpt:trainType'])
# save WomenOnlyCar
if 'odpt:womenOnlyCar' in d:
woc_list = d['odpt:womenOnlyCar']
for woc in woc_list:
fs = Station.get_or_create(woc['odpt:fromStation'])
ts = Station.get_or_create(woc['odpt:toStation'])
time_from = dp.parse(
woc['odpt:availableTimeFrom']).time()
time_until = dp.parse(
woc['odpt:availableTimeUntil']).time()
WomenOnlyCar.get_or_create(rw, fs, ts,
woc['odpt:operationDay'],
time_from, time_until,
woc['odpt:carComposition'],
woc['odpt:carNumber'])
except:
print d
raise
def handle_index(page=1):
args = request.query.decode("utf-8")
page = int(args.page)
searchType = args.searchType
query = args.query
# search
solr = Solr(SOLR_URL + '/' + solr_core[searchType] + '/')
start = (page - 1) * PER_PAGE
solr_query_params = {
'start':start,
'rows':10,
'hl': 'true',
'hl.fl': 'body',
'hl.fragsize': 100,
'hl.snippets': 3,
'df': 'text'
}
responseObj = solr.search(query, **solr_query_params)
ResultObj = parse_response(responseObj)
resultNum = ResultObj['hits']
print resultNum
if resultNum == 0:
return "No match Docs find! What's a pity"
resultNum = resultNum if resultNum < TOTAL else TOTAL
paginator = Pagination(page, PER_PAGE, resultNum)
# result
results = results_for_page(ResultObj, page, PER_PAGE, resultNum)
# ad
if resultNum < 2:
ads = ads_for_page(ResultObj, 1)
else:
ads = ads_for_page(ResultObj, AD_NUM)
#spellcheck
spell_flag = True
query_right = spellcheck.spellCorret(query, wordSet, bagOfword, parameter)
if len(query_right) == 0:
spell_flag = False
template_params = {
'query': query,
'results': results,
'ads': ads,
'paginator': paginator,
'resultNum': resultNum,
'searchType': searchType,
'spellFlag': spell_flag,
'query_right': query_right
}
return template('result.ftl', **template_params)
def import_all(cls):
cls.objects.all().delete()
prm = create_prm(cls.RDF_TYPE)
res = request_metro(prm)
data = parse_response(res)
for d in data:
try:
date = dp.parse(d['dc:date'])
rw = cls(ld_context = d['@context'],
ld_id = d['@id'],
ld_type = d['@type'],
same_as = d['owl:sameAs'],
date = date,
title = d['dc:title'],
operator = d['odpt:operator'],
line_code = d['odpt:lineCode'],
region = d['ug:region']
)
rw.save()
# save StationOrder
so_list = d['odpt:stationOrder']
for so in so_list:
st = Station.get_or_create(so['odpt:station'])
StationOrder.get_or_create(rw, st, so['odpt:index'])
# save TravelTime
tt_list = d['odpt:travelTime']
for tt in tt_list:
fs = Station.get_or_create(tt['odpt:fromStation'])
ts = Station.get_or_create(tt['odpt:toStation'])
TravelTime.get_or_create(rw, fs, ts, tt['odpt:necessaryTime'], tt['odpt:trainType'])
# save WomenOnlyCar
if 'odpt:womenOnlyCar' in d:
woc_list = d['odpt:womenOnlyCar']
for woc in woc_list:
fs = Station.get_or_create(woc['odpt:fromStation'])
ts = Station.get_or_create(woc['odpt:toStation'])
time_from = dp.parse(woc['odpt:availableTimeFrom']).time()
time_until = dp.parse(woc['odpt:availableTimeUntil']).time()
WomenOnlyCar.get_or_create(rw, fs, ts, woc['odpt:operationDay'], time_from, time_until,
woc['odpt:carComposition'], woc['odpt:carNumber'])
except:
print d
raise
def import_all(cls):
prm = create_prm(cls.RDF_TYPE)
res = request_metro(prm)
data = parse_response(res)
instances = []
for d in data:
try:
date = dp.parse(d['dc:date'])
valid = dp.parse(d['dct:valid'])
from_station = Station.get_or_create(d['odpt:fromStation'])
starting_station = Station.get_or_create(d['odpt:startingStation'])
terminal_station = Station.get_or_create(d['odpt:terminalStation'])
to_station = Station.get_or_create(d['odpt:toStation'])
cls(ld_context = d['@context'],
ld_id = d['@id'],
ld_type = d['@type'],
date = date,
valid = valid,
delay = d['odpt:delay'],
frequency = d['odpt:frequency'],
from_station = from_station,
rail_direction = d['odpt:railDirection'],
railway = d['odpt:railway'],
starting_station = starting_station,
terminal_station = terminal_station,
to_station = to_station,
train_number = d['odpt:trainNumber'],
train_owner = d['odpt:trainOwner'],
train_type = d['odpt:trainType'],
same_as = d['owl:sameAs'],
from_station_raw = d['odpt:fromStation'],
starting_station_raw = d['odpt:startingStation'],
terminal_station_raw = d['odpt:terminalStation'],
to_station_raw = d['odpt:toStation'],
).save()
except:
print sys.exc_info()
print d
def import_all(cls):
cls.objects.all().delete()
prm = create_prm(cls.RDF_TYPE)
res = request_metro(prm)
data = parse_response(res)
instances = []
for d in data:
try:
date = dp.parse(d["dc:date"])
conn_railways = ",".join(d["odpt:connectingRailway"])
exits = ",".join(d["odpt:exit"]) if d["odpt:exit"] else ""
surveys = ",".join(d["odpt:passengerSurvey"])
instances.append(
cls(
ld_context=d["@context"],
ld_id=d["@id"],
ld_type=d["@type"],
same_as=d["owl:sameAs"],
date=date,
title=d["dc:title"],
connecting_railways=conn_railways,
exits=exits,
facility=d["odpt:facility"],
operator=d["odpt:operator"],
passenger_surveys=surveys,
railway_id=d["odpt:railway"],
station_code=d["odpt:stationCode"],
region=d["ug:region"],
)
)
except:
print d
raise
for ins in instances:
ins.save()
def system_eval():
global trec_eval_results,baseline
baseline = OrderedDict(baseline)
if len(trec_eval_results) != 0:
template_params = {
'trec_eval_results': trec_eval_results,
'baseline': baseline
}
return template('syseval_result.ftl', **template_params)
trec_list = ['Trec14', 'Trec09']
R = lambda x:int(filter(str.isdigit,x))
for trec in trec_list:
qrels = trec_dict[trec]['qrels.adhoc']
query_dict = evaluate.get_query_info(trec_dict[trec]['queries'])
print 'query len %d' % len(query_dict)
core = 'Trec09' if R(trec) < 12 else 'Trec12'
solr = Solr(SOLR_URL + '/' + solr_core[core] + '/', timeout=100)
results = []
for query_id, query in query_dict.items():
responseObj = solr.search(query, **solr_query_params)
ResultObj = parse_response(responseObj)
results.append([query_id, ResultObj])
# gen submit and save
evaluate.gen_save_submmit(results, RUN_FILE_PATH)
# eval by eval_program
cmd = EVAL_EXE + ' -m all_trec -M 1000 ' + qrels + ' ' + RUN_FILE_PATH + ' > ' + RESULT_PATH
exe_cmd(cmd)
# read eval result and display
eval_results = evaluate.read_eval(RESULT_PATH)
trec_eval_results[trec] = eval_results
print trec + ' is Done!'
template_params = {
'trec_eval_results': trec_eval_results,
'baseline': baseline
}
return template('syseval_result.ftl', **template_params)
