def run_search(self, startDate, track, endDate, writeDir):
'''
Main searching method. This will set off a scrape that will end with
the report being written.
'''
#List for the race objects
meetingResults = []
#Set up a racedate datetime object for comparing dates
startDateObj = self.build_date_obj(startDate)
if endDate == "1":
endDateObj = startDateObj
endDateObj += datetime.timedelta(days=1)
else:
endDateObj = self.build_date_obj(endDate)
dates = self.get_dates(startDateObj, endDateObj)
#Get urls to query
queryUrls = self.get_query_urls(dates, track)
#print queryUrls
for t, urls in queryUrls.iteritems():
for u in urls:
content = self.get_page_content(u)
if content is None:
continue
trackNameEls, goingAllowances, forecasts, tricasts = \
self.get_track_info_elements(content, t)
#We can assume that our track name matches.
#Scrape the race and date
for name, allowance, fcast, tcast in zip(
trackNameEls, goingAllowances, forecasts, tricasts):
raceDate, race, grade, distance, goingAllowance, \
forecast,tricast, dogs = self.get_race_info(name,
allowance, fcast, tcast)
raceResult = Race(t, raceDate, race, grade, distance,
goingAllowance, forecast, tricast, dogs)
meetingResults.append(raceResult)
generator = ReportGenerator(os.getcwd(), writeDir)
generator.generate_report(meetingResults)
meetingResults = []
if len(urls) > 1:
#Don't get too greedy
print("Going to sleep for 5 seconds")
sleep(5)
class Checkstyle:
def __init__(self, strategy):
self.report_generator = ReportGenerator(strategy)
@timing
def perform_checkstyle(self, inspected_dir):
issues = []
for root, directories, file_names in os.walk(inspected_dir):
for filename in file_names:
file_name_match = GOSU_FILE_NAME_REG.search(filename)
if file_name_match:
#use one of below
inspector = FileInspector(root, filename)
#inspector = BlamingFileInspector(root, filename)
issues += inspector.inspect()
self.report_generator.generate(issues)
def __init__(self):
self.func_table = {
"SetParam": self._invoke_set_params,
"Generate": self._invoke_report_generate
}
self.backend_entry_point = SettingMapper.query.get(
'backend_entry_point').value
# Windowsとそれ以外でマウント先を変更する
if os.name == 'nt':
mount_dst_path = Path(
SettingMapper.query.get('mount_src_path').value)
else:
mount_dst_path = Path(
SettingMapper.query.get('mount_dst_path').value)
self.report_home_path = mount_dst_path / 'report'
self._initialize_report_dir(self.report_home_path)
self.report_generator = ReportGenerator(
home_path=str(self.report_home_path) + os.sep)
self.backend_report_home = mount_dst_path / 'backend' / 'report_gen'
def __init__(self, strategy):
self.report_generator = ReportGenerator(strategy)
class ReportGeneratorService:
def __init__(self):
self.func_table = {
"SetParam": self._invoke_set_params,
"Generate": self._invoke_report_generate
}
self.backend_entry_point = SettingMapper.query.get(
'backend_entry_point').value
# Windowsとそれ以外でマウント先を変更する
if os.name == 'nt':
mount_dst_path = Path(
SettingMapper.query.get('mount_src_path').value)
else:
mount_dst_path = Path(
SettingMapper.query.get('mount_dst_path').value)
self.report_home_path = mount_dst_path / 'report'
self._initialize_report_dir(self.report_home_path)
self.report_generator = ReportGenerator(
home_path=str(self.report_home_path) + os.sep)
self.backend_report_home = mount_dst_path / 'backend' / 'report_gen'
def _initialize_report_dir(self, path):
# workdirを作成
if path.exists():
remove_tree(str(path))
path.mkdir(parents=True, exist_ok=True)
# templateをコピー
copy_src_dir = Path(__file__).joinpath('../../../report/template')
copy_dst_dir = path / 'template'
if copy_dst_dir.exists():
remove_tree(str(copy_dst_dir))
copy_tree(src=str(copy_src_dir.resolve()),
dst=str(copy_dst_dir.resolve()))
def _invoke_set_params(self,
request: PostReportGeneratorReq,
_=None) -> {}:
# 先頭のdestinationのみ反映
td_id = int(request.destination[0])
td = TestDescriptionMapper.query.get(td_id)
if td.run is not None:
for td_graph in td.run.graphs:
param_graphs = [
g for g in request.params.graphs if g.id_ == td_graph.id
]
if len(param_graphs) > 0:
param_graph = param_graphs[0]
td_graph.report_required = param_graph.report_required
if param_graph.report_required:
td_graph.report_index = param_graph.report_index
td_graph.report_name = param_graph.report_name
else:
td_graph.report_required = False
if request.params.opinion is not None:
td.opinion = request.params.opinion
sql_db.session.commit()
def _invoke_report_generate(
self,
request: PostReportGeneratorReq,
test_descriptions: [TestDescriptionMapper] = None) -> {}:
# 事前処理
# フォルダ準備
dt_now_jst = datetime.now(timezone(
timedelta(hours=9))).strftime('%Y%m%d%H%M%S')
base_dir = self.backend_report_home / dt_now_jst
in_dir = base_dir / 'in'
in_dir.mkdir(parents=True)
out_dir = base_dir / 'out'
out_dir.mkdir(parents=True)
# 入力JSON作成
in_json = {}
target_td_ids = []
# 以下条件を除外したtarget_td_idsを作る
# 既に削除済み, 未実行（None）、実行時失敗（ERR）
if len(request.destination) == 0:
# test_descriptionsは既に削除済みTDを除外したリスト
target_td_ids = [
td.id for td in test_descriptions
if td.run and td.run.result != 'ERR'
]
else:
tmp_td_ids = [int(td_id) for td_id in request.destination]
for td_id in tmp_td_ids:
td = TestDescriptionMapper.query\
.filter(TestDescriptionMapper.id == td_id)\
.filter(TestDescriptionMapper.delete_flag == False).first()
if td.run and td.run.result != 'ERR':
target_td_ids.append(td_id)
if len(target_td_ids) == 0:
raise QAINotFoundException(
'D14004', 'these test description is not running')
file_path_list = []
type_list = []
quality_props_list = []
td_id__list = []
required_list = []
report_name = []
for td_id in target_td_ids:
td = TestDescriptionMapper.query.get(td_id)
if td.run_id is None:
raise QAINotFoundException(
'D14002', 'test description\'s result is None')
# opinionファイル出力
if len(td.opinion) != 0:
opinion_path = in_dir / ('opinion' + str(td_id) + ".txt")
with open(str(opinion_path), mode='w', encoding='utf-8') as f:
f.write(td.opinion)
file_path_list.append(str(opinion_path))
type_list.append('text')
quality_props_list.append(td.quality_dimension_id)
td_id__list.append(str(td_id))
required_list.append(True)
report_name.append('Opinion')
graphs = GraphMapper.query.\
filter(GraphMapper.run_id == td.run_id).\
filter(GraphMapper.report_required == True).\
order_by(asc(GraphMapper.report_index)).\
all()
for graph in graphs:
file_path_list.append(graph.download.path)
type_list.append(graph.graph_template.resource_type.type)
quality_props_list.append(td.quality_dimension_id)
td_id__list.append(str(td_id))
required_list.append(graph.report_required)
report_name.append(graph.report_name)
in_json['filepath'] = dict(
zip(range(len(file_path_list)), file_path_list))
in_json['type'] = dict(zip(range(len(type_list)), type_list))
in_json['quality_props'] = dict(
zip(range(len(quality_props_list)), quality_props_list))
in_json['testDescriptionID'] = dict(
zip(range(len(td_id__list)), td_id__list))
in_json['required'] = dict(
zip(range(len(required_list)), required_list))
in_json['name'] = dict(zip(range(len(report_name)), report_name))
in_json_path = in_dir / 'input.json'
with open(str(in_json_path), 'w', encoding='utf-8') as f:
json.dump(in_json, f, indent=4, ensure_ascii=False)
# レポート生成
pdf_file_path = self.report_home_path / 'work' / 'report.pdf'
pdf_file = self.report_generator.report_generate(
sql_db, str(in_json_path), str(pdf_file_path))
if not pdf_file or not Path(pdf_file).exists():
raise QAINotFoundException('D16000', 'failed report generate')
# 事後処理
res = {}
try:
dst_path = out_dir / Path(pdf_file).name
shutil.copy(src=pdf_file, dst=str(dst_path))
dl = DownloadMapper(path=pdf_file)
sql_db.session.add(dl)
sql_db.session.commit()
res['ReportUrl'] = self.backend_entry_point + '/download/' + str(
dl.id)
except Exception as e:
print('Exception: {}'.format(e))
sql_db.session.rollback()
raise e
return res
@log(logger)
def post(self, organizer_id: str, ml_component_id: int,
request: PostReportGeneratorReq) -> PostReportGeneratorRes:
test = TestMapper.query.\
filter(TestMapper.ml_component_id == ml_component_id).\
filter(MLComponentMapper.org_id == organizer_id).first()
if test is None:
raise QAINotFoundException('D14000', 'not found test descriptions')
if request.command not in self.func_table:
raise QAIBadRequestException('D10001', 'invaid command')
# delete_flagがTrueのTDを除外したTestDescriptionMapperを作る
mapper = TestDescriptionMapper.query. \
filter(TestDescriptionMapper.test_id == test.id). \
filter(TestDescriptionMapper.delete_flag == False). \
all()
if not mapper:
raise QAINotFoundException('D14001',
'test descriptions are all deleted')
try:
func = self.func_table[request.command]
out_params = func(request, mapper)
except Exception as e:
print('Exception: {}'.format(e))
sql_db.session.rollback()
raise e
return PostReportGeneratorRes(result=Result(
code='D12000', message="command invoke success."),
out_params=out_params)