def get_top_performers_from_random_population(env, population, elite_size):
evaluated_population = [(model, evaluate(model, env))
for model in population]
evaluated_population.sort(key=lambda result: result[1], reverse=True)
elite, top_scores = unzip(evaluated_population[:elite_size])
for e in elite:
e.share_memory()
return elite, np.array(top_scores)
def Query():
queryset = Report.objects.filter(status=0)
inprocess_queryset = Report.objects.filter(status=1)
in_length = len(inprocess_queryset)
for report in queryset:
if (in_length < 5):
report.status = 1
#report.save()
print(report.alpha_name, report.file)
in_length += 1
flag = False
utils.unzip(report)
if (utils.validate_files(report)):
try:
flag = utils.compile_alpha(report)
except RuntimeError as e:
print(e)
print('catch')
#utils.clean()
if (flag == True):
report.status = 2
else:
report.status = 3
def get_ghs_data(ghs_type, config, country_iso3, input_dir):
"""
:param ghs_type: One of either "SMOD" or "POP"
"""
logger.info(f'Getting GHS data for {ghs_type}')
download_dir = os.path.join(input_dir, GHS_DIR, 'zip')
Path(download_dir).mkdir(parents=True, exist_ok=True)
for column, row in [
ast.literal_eval(x) for x in config['column_row_pairs']
]:
zip_filename = os.path.join(download_dir,
f'{ghs_type}_2015_1km_{column}_{row}.zip')
utils.download_url(
f'{GHS_URL_BASE}/{GHS_URL[ghs_type].format(column=column, row=row)}',
zip_filename)
utils.unzip(zip_filename, download_dir)
# Make a mosaic
files_to_mosaic = [
rasterio.open(f)
for f in glob.glob(os.path.join(download_dir, f'*_{ghs_type}_*.tif'))
]
logger.info(f'Making mosiac of {len(files_to_mosaic)} files')
mosaic, out_trans = merge(files_to_mosaic)
out_meta = files_to_mosaic[0].meta.copy()
out_meta.update({
"driver": "GTiff",
"height": mosaic.shape[1],
"width": mosaic.shape[2],
"transform": out_trans
})
output_filepath = os.path.join(
input_dir, GHS_DIR,
OUTPUT_GHS[ghs_type].format(country_iso3=country_iso3))
with rasterio.open(output_filepath, "w", **out_meta) as dest:
dest.write(mosaic)
logger.info(f'Wrote file to {output_filepath}')
def Query(pk):
report = Report.objects.get(report_id=pk)
print('jobs {} running....'.format(pk))
report.status = 1
report.save()
flag = False
utils.unzip(report)
if (utils.validate_files(report)):
try:
flag, submitted_status = utils.compile_alpha(report)
except RuntimeError as e:
report.error_message = u"编译错误"
if (flag == True):
if submitted_status == 1:
report.status = 4
else:
report.status = 2
else:
report.status = 3
else:
report.error_message = u"解压错误或文件名错误"
report.status = 3
report.save()
print('jobs[ts_id=%s] done' % report.report_id)
def Query(pk):
report = Report.objects.get(report_id=pk)
print('jobs {} running....'.format(pk))
report.status = 1
report.save()
flag = False
utils.unzip(report)
if (utils.validate_files(report)):
try:
flag,submitted_status = utils.compile_alpha(report)
except RuntimeError as e:
report.error_message = u"编译错误"
if (flag == True):
if submitted_status == 1:
report.status = 4
else:
report.status = 2
else:
report.status = 3
else:
report.error_message = u"解压错误或文件名错误"
report.status = 3
report.save()
print('jobs[ts_id=%s] done' % report.report_id)
def ftpDownload(case):
'''
入库授权路径：storage/auth/year/month/day/hour/申请号/文献类型/abc.zip
:param case:
:return:
'''
shenqingh = case['shenqingh']
documentType = case['wenjianlx']
fid = case['fid']
url = case['url']
submitDate = case['submitDate']
user, passwd, host, port, fileName = getFtpInfo(url)
fileIndex = fileName.rfind('/')
fileName2 = fileName[fileIndex + 1:]
ftp = FTP()
try:
#1.创建授权文件夹
fileDir = getAuthDir(shenqingh, documentType, submitDate, fid)
file = fileDir + '/' + fileName2
#1.下载
ftp.connect(host, int(port))
ftp.login(user, passwd)
with open(file, 'wb') as f:
ftp.retrbinary('RETR ' + fileName, f.write)
ftp.quit()
#2.解压到原始目录下
success = utils.unzip(file, fileDir)
if success == 0:
return False
files = glob.glob(fileDir + '/*.xml')
if len(files) != 1:
logger.info("shenqingh" + str(shenqingh) +
",xml文件个数不为1,删除目录，路径为：" + fileDir)
if os.path.exists(fileDir):
shutil.rmtree(fileDir)
return False
#3.删除压缩包
#4.将原始目录中的图片生成缩略图到原图同一文件夹下
for thumb in thumbSizeList:
thrumbImgs(fileDir, fileDir, thumb[0], thumb[1])
except Exception as e:
logger.info("shenqingh download exception: " + str(shenqingh) +
traceback.format_exc() + '---' + str(case))
return False
return True
def ftpDownload(case):
shenqingh = case['shenqingh']
documentType = case['wenjianlx']
fid = case['fid']
url = case['url']
submitDate = case['submitDate']
user, passwd, host, port, fileName = getFtpInfo(url)
fileIndex = fileName.rfind('/')
fileName2 = fileName[fileIndex + 1:]
ftp = FTP()
fileDir = utils.createOriginDir(shenqingh, documentType, submitDate, fid)
file = fileDir + '/' + fileName2
try:
#1.下载
ftp.connect(host, int(port))
ftp.login(user, passwd)
with open(file, 'wb') as f:
ftp.retrbinary('RETR ' + fileName, f.write)
#2.解压到原始目录下
success = utils.unzip(file, fileDir)
if success == 0:
return False
#验证xml文件是否唯一
files = glob.glob(fileDir + '/*.xml')
if len(files) != 1:
logger.info("xml文件个数不为1")
if os.path.exists(fileDir):
shutil.rmtree(fileDir)
return False
#3.删除压缩包
#4.将原始目录中的图片生成缩略图到原图同一文件夹下
for thumb in thumbSizeList:
thrumbImgs(fileDir, fileDir, thumb[0], thumb[1])
# #5.复制原始目录到当前版目录下
modifyPath = utils.createModifyDir(shenqingh, documentType, submitDate,
fid)
copyDir(fileDir, modifyPath)
ftp.quit()
except Exception as e:
logger.info(traceback.format_exc())
return False
return True
def unzip_data(self):
if self.configs["data_zip_file"]:
zip_file = self.configs["data_zip_file"]
folder = os.path.dirname(zip_file)
print("Doing unzip file {0}:".format(zip_file))
unzip(zip_file, folder)
def get_top_performers(evaluated_population, elite, elite_size):
sort_population(evaluated_population)
top_ids, top_seeds, top_scores = unzip(evaluated_population[:elite_size])
mutate_models(elite, top_ids, top_seeds)
return np.array(top_scores)
def get_top_performers(evaluated_population, top_count):
sort_population(evaluated_population)
top_models, top_scores = unzip(evaluated_population[:top_count])
return top_models, np.array(top_scores)
appLogic.deleteAll(case)
return
except Exception as e:
appLogic.deleteAll(case)
print traceback.format_exc()
return
print "success!"
def handleZip(case,srcZipFile,desDir,desZipFile):
'''
复制原始包到原始目录，解压，生成缩略图，复制当前目录
:return:
'''
shutil.copyfile(srcZipFile, desZipFile)
utils.unzip(desZipFile, desDir)
# 验证xml文件是否唯一
files = glob.glob(desDir + '/*.xml')
if len(files) != 1:
print 'failed,xml files != 1,delete originDir!,originDir: ' + desDir
shutil.rmtree(desDir)
return 0
# 3.删除压缩包
# 4.将原始目录中的图片生成缩略图到原图同一文件夹下
for thumb in thumbSizeList:
appLogic.thrumbImgs(desDir, desDir, thumb[0], thumb[1])
#5.复制原始目录到当前版目录下
modifyPath = utils.createModifyDir(case['shenqingh'], case['wenjianlx'], case['submitDate'], case['fid'])
appLogic.copyDir(desDir, modifyPath)
return 1