def preprocess_model_server(self, data, data_type):
if data_type == 'URL' or data_type == 'PATH':
data_path = os.path.normpath(data)
data_name = data_path.split('/')[-1]
ext_name = data_path.split('/')[-1].split('.')[-1].lower()
if ext_name in ['jpg', 'jpeg', 'png', 'bmp', 'gif']:
image_data = Image.open(data_path)
img_data = np.fromstring(image_data.tobytes(), dtype=np.uint8)
img_data = img_data.reshape(
(image_data.size[1], image_data.size[0],
len(image_data.getbands())))
return img_data, data_name, 'IMAGE'
elif ext_name in ['mp4', 'avi']:
reader = imageio.get_reader(data_path)
image_list = []
for im in reader:
img_data = np.fromstring(im.tobytes(), dtype=np.uint8)
img_data = img_data.reshape(
(im.shape[0], im.shape[1], im.shape[2]))
image_list.append(np.expand_dims(img_data, 0))
image_volume = np.vstack(image_list)
return image_volume, data_name, 'VIDEO'
elif ext_name in ['txt']:
with open(data_path, 'r') as fp:
content = fp.read()
return content, data_name, 'FILE'
else:
# TODO: support video and sound
logger.warn('dont support file type %s' % ext_name)
return None, None, None
else:
return data, str(uuid.uuid4()), 'STRING'
def random(self, count=1):
# get study configuration
study_configuration = json.loads(self.study.study_configuration)
try_count = 0
proposed_search_space = []
while True:
if try_count > 50:
logger.warn('couldnt find valid graph structure for study %s'%self.study.name)
return [(None, None)]
random_p = random.choice(list(range(self.population_size)))
default_graph_str = study_configuration['searchSpace']['current_population'][random_p]
default_graph = Decoder().decode(default_graph_str)
default_graph.layer_factory = BaseLayerFactory()
default_graph_info = study_configuration['searchSpace']['current_population_info'][random_p]
# 1.step random mutate one cell
graph, graph_info = self.mutation_operator._mutate_for_cell(copy.deepcopy(default_graph),
copy.deepcopy(default_graph_info))
# 2.step skip branch in branch
graph, graph_info = self.mutation_operator._mutate_for_branch(graph, graph_info)
# 3.step random skip once between cells
if random.random() < self.mutation_operator._mutate_rate_for_skip_cell:
for start_layer_id, end_layer_id in self.mutation_operator._find_allowed_skip_cell(graph_info):
try:
graph = self.mutation_operator._mutate_for_skip_cell(graph, start_layer_id, end_layer_id)
break
except:
pass
# 4.step mutation branch in cell
for _ in range(self.branch_num - 1):
for start_layer_id, end_layer_id in self.mutation_operator._find_allowed_skip_branch(graph_info):
try:
graph = self.mutation_operator._mutate_for_skip_branch(graph, start_layer_id, end_layer_id)
break
except:
pass
# graph.visualization('%s.png' % (str(uuid.uuid4())))
graph_dna = self.dna(graph, graph_info)
trials = Trial.filter(study_name=self.study.name)
is_not_valid = False
for t in trials:
if str(t.structure_encoder) == str(graph_dna.tolist()):
is_not_valid = True
break
if is_not_valid:
try_count += 1
continue
proposed_search_space.append((graph_dna, Encoder(skipkeys=True).encode(graph), graph_info))
if len(proposed_search_space) == count:
break
return proposed_search_space
def __vcg_try_get(A, page_url, try_times=5):
while try_times != 0:
Result = A.get(page_url, timeout=7, allow_redirects=True)
if Result.status_code == 200:
return Result
logger.warn("sleep 10s, continue try.")
time.sleep(10)
try_times -= 1
return None
def dispatch_prepare_data(self, data, data_type):
data_path = None
data_name = None
if data_type == 'URL':
# download data
download_path = os.path.join(self.api_static_path, 'input')
if not os.path.exists(download_path):
os.makedirs(download_path)
data_name = os.path.normpath(data).split('/')[-1]
data_name = '%s_%s' % (str(uuid.uuid4()), data_name)
data_path = download(data, os.path.join(self.api_static_path, 'input'), data_name)
data_path = os.path.normpath(data_path)
elif data_type == 'PATH':
data_name = data.split('/')[-1]
if os.path.exists(os.path.join(self.api_static_path, 'input', data_name)):
data_path = os.path.join(self.api_static_path, 'input', data_name)
if data_type == 'URL' or data_type == 'PATH':
ext_name = data_path.split('/')[-1].split('.')[-1].lower()
if ext_name in ['jpg', 'jpeg', 'png', 'bmp']:
image_data = Image.open(data_path)
img_data = np.fromstring(image_data.tobytes(), dtype=np.uint8)
img_data = img_data.reshape((image_data.size[1], image_data.size[0], len(image_data.getbands())))
fsize = os.path.getsize(data_path)
fsize = fsize / float(1024 * 1024)
return img_data, data_name, 'IMAGE', round(fsize, 2), {}
elif ext_name in ['mp4']:
# TODO: bug
reader = imageio.get_reader(data_path)
image_list = []
for im in reader:
img_data = np.fromstring(im.tobytes(), dtype=np.uint8)
img_data = img_data.reshape((im.shape[0], im.shape[1], im.shape[2]))
image_list.append(np.expand_dims(img_data, 0))
image_volume = np.vstack(image_list)
fsize = os.path.getsize(data_path)
fsize = fsize / float(1024 * 1024)
return image_volume, data_name, 'VIDEO', round(fsize,2), {'FPS': reader.get_meta_data()['fps']}
else:
# TODO: support video and sound
logger.warn('dont support file type %s' % ext_name)
return None, None, None, 0
def _pick_idle_port(from_port=40000, check_count=100):
check_port = from_port
while check_count:
if not _is_open('127.0.0.1', check_port):
break
logger.warn('port %d is occupied, try to use %d port'%(int(check_port), int(check_port + 1)))
check_port += 1
check_count -= 1
if check_count == 0:
check_port = None
if check_port is None:
logger.warn('couldnt find valid free port')
exit(-1)
return check_port
def __vcg_find_and_download(waiting_process_queue, save_dir):
A = requests.Session()
A.headers = headers
query_url = 'https://www.vcg.com/creative-image/{}'.format(keyword)
if download_data_type == 'video':
query_url = 'https://www.vcg.com/creative-video-search/{}'.format(
keyword)
download_count = 0
content = None
nav_page_list = []
Result = A.get(query_url, timeout=7, allow_redirects=True)
if Result.status_code != 200:
logger.error("%s couldnt connect." % query_url)
return
content = Result.text
soup = BeautifulSoup(content)
# 1.step 分析待检索页面
if download_data_type == 'image':
pages = soup.findAll("a", class_="paginationClickBox")
last_page = pages[-1]
total_page = soup.findAll("span", class_="paginationTotal")
try:
total_page_num = int(last_page.text)
if len(total_page) > 0:
total_page_num = (int)(total_page[0].text[1:-1])
except:
logger.error("Fail to parse nav page, use default 2.")
total_page_num = 2
page_nav_url = last_page.get('href').split('?')[0]
for i in range(2, total_page_num + 1):
nav_page_list.append('%s?page=%d' % (page_nav_url, i))
else:
pages = soup.findAll("a", class_="_2IlL4")
last_page = pages[-2]
try:
total_page_num = int(last_page.text)
except:
logger.error("Fail to parse nav page, use default 2.")
total_page_num = 2
page_nav_url = last_page.get('href').split('?')[0]
for i in range(2, total_page_num + 1):
nav_page_list.append('%s?page=%d' % (page_nav_url, i))
# 2.step 分析当前页面的图像
logger.info("Analyze nav page(%d/%d) %s" %
(1, len(nav_page_list), query_url))
img_url_list = []
if download_data_type == 'image':
# 分析图像资源
img_list = soup.findAll("img", class_="lazyload_hk")
img_url_list = [
"http://{}".format(p.get("data-src")[2:]) for p in img_list
]
else:
# 分析视频资源
img_list = soup.findAll('source', type="image/webp")
img_url_list = []
video_name_list = []
for p in img_list:
kk = p.get('data-srcset')[2:].split('/')[3].split(
'?')[0].split('_')[0]
if kk not in video_name_list:
video_name_list.append(kk)
img_url_list.append(
'http://gossv.cfp.cn/videos/mts_videos/medium/temp/{}.mp4'
.format(kk))
for img_url in img_url_list:
__vcg_img_download(waiting_process_queue, save_dir, img_url,
keyword, download_count)
download_count += 1
# 3.step 继续分析所有等待导航页面
for page_index, page_url in enumerate(nav_page_list):
logger.info("Analyze nav page(%d/%d) %s" %
(page_index + 2, len(nav_page_list), page_url))
Result = A.get(page_url, timeout=7, allow_redirects=True)
if Result.status_code != 200:
logger.warn("Couldnt connect and analyze %s. (page %s)" %
((Result.text, page_url)))
Result = __vcg_try_get(A, page_url, 5)
if Result is None:
logger.warn("Couldnt connect %s, return." % page_url)
return
content = Result.text
soup = BeautifulSoup(content)
img_url_list = []
if download_data_type == 'image':
# 分析图像资源
img_list = soup.findAll("img", class_="lazyload_hk")
img_url_list = [
"http://{}".format(p.get("data-src")[2:]) for p in img_list
]
else:
# 分析视频资源
img_list = soup.findAll('source', type="image/webp")
img_url_list = []
video_name_list = []
for p in img_list:
kk = p.get('data-srcset')[2:].split('/')[3].split(
'?')[0].split('_')[0]
if kk not in video_name_list:
video_name_list.append(kk)
img_url_list.append(
'http://gossv.cfp.cn/videos/mts_videos/medium/temp/{}.mp4'
.format(kk))
logger.info("Finding %d data" % len(img_url_list))
for img_url in img_url_list:
__vcg_img_download(waiting_process_queue, save_dir, img_url,
keyword, download_count)
download_count += 1
# 添加结束标记
if waiting_process_queue is not None:
waiting_process_queue.put(None)
def vcg_download(keyword,
download_params,
download_save_dir,
process_queue=None):
# 视觉中国
headers = {
'Accept-Language':
'zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2',
'Connection': 'keep-alive',
'User-Agent':
'Mozilla/5.0 (X11; Linux x86_64; rv:60.0) Gecko/20100101 Firefox/60.0',
'Upgrade-Insecure-Requests': '1'
}
download_data_type = 'image'
if download_params is not None:
download_data_type = download_params.get('download_data_type', 'image')
if download_data_type not in ['image', 'video']:
logger.warn('Only support donwload data_type: image or video')
download_data_type = 'image'
def __vcg_img_download(waiting_process_queue, save_dir, img_url, keyword,
count):
try:
logger.info("Downloading(%d) %s." % (count + 1, img_url))
pic = requests.get(img_url, timeout=7)
except BaseException:
logger.error("Couldnt download %s." % img_url)
return
else:
file_prefix = 'VCG_' + keyword + '_' + str(count)
file_name = file_prefix + '.jpg' if download_data_type == 'image' else file_prefix + '.mp4'
file_path = os.path.join(save_dir, file_name)
fp = open(file_path, 'wb')
fp.write(pic.content)
fp.close()
logger.info("Finish download %s ." % img_url)
if waiting_process_queue is not None:
waiting_process_queue.put(file_path)
def __vcg_try_get(A, page_url, try_times=5):
while try_times != 0:
Result = A.get(page_url, timeout=7, allow_redirects=True)
if Result.status_code == 200:
return Result
logger.warn("sleep 10s, continue try.")
time.sleep(10)
try_times -= 1
return None
def __vcg_find_and_download(waiting_process_queue, save_dir):
A = requests.Session()
A.headers = headers
query_url = 'https://www.vcg.com/creative-image/{}'.format(keyword)
if download_data_type == 'video':
query_url = 'https://www.vcg.com/creative-video-search/{}'.format(
keyword)
download_count = 0
content = None
nav_page_list = []
Result = A.get(query_url, timeout=7, allow_redirects=True)
if Result.status_code != 200:
logger.error("%s couldnt connect." % query_url)
return
content = Result.text
soup = BeautifulSoup(content)
# 1.step 分析待检索页面
if download_data_type == 'image':
pages = soup.findAll("a", class_="paginationClickBox")
last_page = pages[-1]
total_page = soup.findAll("span", class_="paginationTotal")
try:
total_page_num = int(last_page.text)
if len(total_page) > 0:
total_page_num = (int)(total_page[0].text[1:-1])
except:
logger.error("Fail to parse nav page, use default 2.")
total_page_num = 2
page_nav_url = last_page.get('href').split('?')[0]
for i in range(2, total_page_num + 1):
nav_page_list.append('%s?page=%d' % (page_nav_url, i))
else:
pages = soup.findAll("a", class_="_2IlL4")
last_page = pages[-2]
try:
total_page_num = int(last_page.text)
except:
logger.error("Fail to parse nav page, use default 2.")
total_page_num = 2
page_nav_url = last_page.get('href').split('?')[0]
for i in range(2, total_page_num + 1):
nav_page_list.append('%s?page=%d' % (page_nav_url, i))
# 2.step 分析当前页面的图像
logger.info("Analyze nav page(%d/%d) %s" %
(1, len(nav_page_list), query_url))
img_url_list = []
if download_data_type == 'image':
# 分析图像资源
img_list = soup.findAll("img", class_="lazyload_hk")
img_url_list = [
"http://{}".format(p.get("data-src")[2:]) for p in img_list
]
else:
# 分析视频资源
img_list = soup.findAll('source', type="image/webp")
img_url_list = []
video_name_list = []
for p in img_list:
kk = p.get('data-srcset')[2:].split('/')[3].split(
'?')[0].split('_')[0]
if kk not in video_name_list:
video_name_list.append(kk)
img_url_list.append(
'http://gossv.cfp.cn/videos/mts_videos/medium/temp/{}.mp4'
.format(kk))
for img_url in img_url_list:
__vcg_img_download(waiting_process_queue, save_dir, img_url,
keyword, download_count)
download_count += 1
# 3.step 继续分析所有等待导航页面
for page_index, page_url in enumerate(nav_page_list):
logger.info("Analyze nav page(%d/%d) %s" %
(page_index + 2, len(nav_page_list), page_url))
Result = A.get(page_url, timeout=7, allow_redirects=True)
if Result.status_code != 200:
logger.warn("Couldnt connect and analyze %s. (page %s)" %
((Result.text, page_url)))
Result = __vcg_try_get(A, page_url, 5)
if Result is None:
logger.warn("Couldnt connect %s, return." % page_url)
return
content = Result.text
soup = BeautifulSoup(content)
img_url_list = []
if download_data_type == 'image':
# 分析图像资源
img_list = soup.findAll("img", class_="lazyload_hk")
img_url_list = [
"http://{}".format(p.get("data-src")[2:]) for p in img_list
]
else:
# 分析视频资源
img_list = soup.findAll('source', type="image/webp")
img_url_list = []
video_name_list = []
for p in img_list:
kk = p.get('data-srcset')[2:].split('/')[3].split(
'?')[0].split('_')[0]
if kk not in video_name_list:
video_name_list.append(kk)
img_url_list.append(
'http://gossv.cfp.cn/videos/mts_videos/medium/temp/{}.mp4'
.format(kk))
logger.info("Finding %d data" % len(img_url_list))
for img_url in img_url_list:
__vcg_img_download(waiting_process_queue, save_dir, img_url,
keyword, download_count)
download_count += 1
# 添加结束标记
if waiting_process_queue is not None:
waiting_process_queue.put(None)
# 搜索和下载
if not os.path.exists(os.path.join(download_save_dir, 'test')):
os.makedirs(os.path.join(download_save_dir, 'test'))
if process_queue is not None:
t = threading.Thread(target=__vcg_find_and_download,
args=(process_queue,
os.path.join(download_save_dir, 'test')))
t.start()
else:
__vcg_find_and_download(process_queue,
os.path.join(download_save_dir, 'test'))
def get_new_suggestions(self, number=1, **kwargs):
# 1.step get current population tag from study configuration
study_configuration = json.loads(self.study.study_configuration)
current_population_tag = int(
study_configuration['searchSpace']['current_population_tag'])
# 2.step fix failed trial (replace those failed trials)
trials = Trial.filter(study_name=self.study.name,
tag=current_population_tag)
for trial in trials:
if trial.status == 'Failed':
logger.warn('trail (id %s) is error and rebuilded' %
(trial.name))
# generate new individual
graph_encoder_str, graph_info = self.random()
trial.structure = [graph_encoder_str, graph_info]
trial.structure_encoder = None
trial.name = '%s-%s' % (str(
uuid.uuid4()), datetime.fromtimestamp(
timestamp()).strftime('%Y%m%d-%H%M%S-%f'))
temp_graph = Decoder().decode(graph_encoder_str)
temp_graph.update_by(graph_info)
trial.multi_objective_value = [1.0 / temp_graph.flops]
trial.status = None
# 3.step get candidate trials of study
candidate_trails = Trial.filter(study_name=self.study.name,
tag=current_population_tag,
status=None)
if len(candidate_trails) == 0:
uncompleted_trails = Trial.filter(study_name=self.study.name,
tag=current_population_tag,
status="UnCompleted")
if len(uncompleted_trails) > 0:
# study not stop, all free worker should wait
return None
while len(candidate_trails) == 0:
# 3.1.step incubate next generation population
elite_population = Population()
completed_trials = Trial.filter(study_name=self.study.name,
tag=current_population_tag,
status="Completed")
for t in completed_trials:
me = self.evolution_control.problem.generateIndividual()
me.id = t.name
me.features = [
Decoder().decode(t.structure[0]), t.structure[1]
]
me.type = 'parent'
me.objectives[0] = t.objective_value
me.objectives[1] = t.multi_objective_value[0]
self.evolution_control.problem.calculate_objectives(me)
elite_population.population.append(me)
if current_population_tag >= 1:
grandpa_population = Population()
grandpa_completed_trials = Trial.filter(
study_name=self.study.name,
tag=current_population_tag - 1,
status="Completed")
for t in grandpa_completed_trials:
me = self.evolution_control.problem.generateIndividual()
me.id = t.name
me.features = [
Decoder().decode(t.structure[0]), t.structure[1]
]
me.type = 'parent'
me.objectives[0] = t.objective_value
me.objectives[1] = t.multi_objective_value[0]
self.evolution_control.problem.calculate_objectives(me)
grandpa_population.population.append(me)
elite_population = self.evolution_control.evolve(
grandpa_population, elite_population)
# cubate next generation by elite population
offspring_population = self.evolution_control.create_children(
elite_population)
current_population_tag += 1
# 3.2.step update trials
study_current_population = []
study_current_population_info = []
for p in offspring_population.population:
trail_name = '%s-%s' % (str(
uuid.uuid4()), datetime.fromtimestamp(
timestamp()).strftime('%Y%m%d-%H%M%S-%f'))
trial = Trial.create(
Trial(self.study.name,
trail_name,
created_time=time.time(),
updated_time=time.time()))
trial.structure = [
Encoder(skipkeys=True).encode(p.features[0]), p.features[1]
]
trial.structure_encoder = None
temp_graph = Decoder().decode(trial.structure[0])
temp_graph.update_by(trial.structure[1])
trial.objective_value = -1
trial.multi_objective_value = [1.0 / temp_graph.flops]
trial.tag = current_population_tag
study_current_population.append(trial.structure[0])
study_current_population_info.append(trial.structure[1])
# update study configuration
study_configuration['searchSpace'][
'current_population'] = study_current_population
study_configuration['searchSpace'][
'current_population_info'] = study_current_population_info
study_configuration['searchSpace'][
'current_population_tag'] = current_population_tag
# regenerate candidate trials
candidate_trails = Trial.filter(study_name=self.study.name,
tag=current_population_tag,
status=None)
self.study.study_configuration = json.dumps(study_configuration)
trial_suggestion = random.choice(candidate_trails)
trial_suggestion.status = 'UnCompleted'
return [trial_suggestion]
def random(self, count=1):
try_count = 0
proposed_search_space = []
while True:
if try_count > 50:
logger.warn('couldnt find valid graph structure for study %s' %
self.study.name)
return [(None, None)]
# 1.step make a structure suggestion
clone_graph = copy.deepcopy(self.graph)
outputs = clone_graph.get_input()
output_node_id = -1
decoder_output_last = []
graph_encoder = []
for output_index, output_id in enumerate(outputs):
branch_offset = (1 + 8 * 8 + 4 * 4 + self.branches)
graph_output_encoder = np.zeros(
(branch_offset * self.branches), dtype=np.float32)
if output_index > 0:
if random.random() > 0.5:
graph_encoder.extend(graph_output_encoder.tolist())
continue
output = clone_graph.node_list[output_id]
temp = [output_id]
for node_id in decoder_output_last:
if clone_graph.node_list[node_id].shape[1] != output.shape[1] or \
clone_graph.node_list[node_id].shape[2] != output.shape[2]:
output_node_id = clone_graph.add_layer(
clone_graph.layer_factory.bilinear_resize(
height=output.shape[1], width=output.shape[2]),
node_id)
temp.append(output_node_id)
else:
temp.append(output_node_id)
if len(temp) > 1:
output_node_id = clone_graph.add_layer(
clone_graph.layer_factory.concat(), temp)
X = [output_node_id]
else:
output_node_id = temp[0]
X = temp
for branch_index in range(self.branches):
# random select branch input from X
X_index_list = list(range(len(X)))
X_select_index_list = sample(
X_index_list, random.randint(1, len(X_index_list)))
X_selected = [X[i] for i in X_select_index_list]
# concat all input
if len(X_selected) > 1:
output_node_id = clone_graph.add_layer(
clone_graph.layer_factory.concat(), X_selected)
else:
output_node_id = X_selected[0]
# encoder 1.step connect
graph_output_encoder[branch_index * branch_offset:(
branch_index +
1) * branch_offset][X_select_index_list] = 1.0
# operator space
r = random.randint(0, 2)
if r == 0:
# 1x1 convolution (conv + bn + relu)
shape = clone_graph.node_list[output_node_id].shape
output_node_id = clone_graph.add_layer(
clone_graph.layer_factory.conv2d(
input_channel=shape[3],
filters=self.channels,
kernel_size_h=1,
kernel_size_w=1), output_node_id)
output_node_id = clone_graph.add_layer(
clone_graph.layer_factory.bn2d(), output_node_id)
output_node_id = clone_graph.add_layer(
clone_graph.layer_factory.relu(), output_node_id)
# encoder 2.step 1x1 convolution
graph_output_encoder[branch_index *
branch_offset:(branch_index + 1) *
branch_offset][self.branches +
0] = 1.0
elif r == 1:
# 3x3 atrous separable convolution
shape = clone_graph.node_list[output_node_id].shape
# rate 1,3,6,9,12,15,18,21
min_hw = min(shape[1], shape[2])
rate_list = [1, 3, 6, 9, 12, 15, 18, 21]
rate_list = [
rate_list[i] for i in range(len(rate_list))
if rate_list[i] < min_hw
]
rate_h_index = random.randint(0, len(rate_list) - 1)
rate_h = rate_list[rate_h_index]
rate_w_index = random.randint(0, len(rate_list) - 1)
rate_w = rate_list[rate_w_index]
output_node_id = clone_graph.add_layer(
clone_graph.layer_factory.separable_conv2d(
input_channel=shape[3],
filters=self.channels,
kernel_size_h=3,
kernel_size_w=3,
rate_h=rate_h,
rate_w=rate_w), output_node_id)
output_node_id = clone_graph.add_layer(
clone_graph.layer_factory.bn2d(), output_node_id)
output_node_id = clone_graph.add_layer(
clone_graph.layer_factory.relu(), output_node_id)
# encoder 3.step 3x3 atrous separable convolution
graph_output_encoder[branch_index *
branch_offset:(branch_index + 1) *
branch_offset][self.branches + 1 +
rate_h_index *
len(rate_list) +
rate_w_index] = 1.0
else:
# spatial pyramid pooling
shape = clone_graph.node_list[output_node_id].shape
min_hw = min(shape[1], shape[2])
gh = [1, 2, 4, 8]
gh = [n for n in gh if n < min_hw]
grid_h_index = random.randint(0, len(gh) - 1)
grid_h = gh[grid_h_index]
gw = [1, 2, 4, 8]
gw = [n for n in gw if n < min_hw]
grid_w_index = random.randint(0, len(gw) - 1)
grid_w = gw[grid_w_index]
output_node_id = clone_graph.add_layer(
clone_graph.layer_factory.spp(grid_h=grid_h,
grid_w=grid_w),
output_node_id)
# encoder 4.step spp
graph_output_encoder[branch_index *
branch_offset:(branch_index + 1) *
branch_offset][self.branches + 1 +
8 * 8 +
grid_h_index * 4 +
grid_w_index] = 1.0
X.append(output_node_id)
output_node_id = clone_graph.add_layer(
clone_graph.layer_factory.concat(), X[1:])
decoder_output_last.append(output_node_id)
graph_encoder.extend(graph_output_encoder.tolist())
# check flops
if clone_graph.flops > self.flops:
try_count += 1
continue
# check structure is not been checked
trials = Trial.filter(study_name=self.study.name)
is_not_valid = False
for t in trials:
if str(t.structure_encoder) == str(graph_encoder):
is_not_valid = True
break
if is_not_valid:
try_count += 1
continue
proposed_search_space.append(
(np.array(graph_encoder),
Encoder(skipkeys=True).encode(clone_graph)))
if len(proposed_search_space) == count:
break
return proposed_search_space
def asynProcess(self, preprocess_type, data):
if preprocess_type == 'DOWNLOAD':
try:
# 1.step 下载(如果有必要)
pic = requests.get(data['url'], timeout=7)
download_path = os.path.join(self.demo_dump, 'static', 'input',
data['file_name'])
with open(download_path, 'wb') as fp:
fp.write(pic.content)
# 2.step 检查
# 2.1.step 检查文件大小
if 'file_size' in self.demo_constraint:
max_file_size = self.demo_constraint['file_size']
fsize = os.path.getsize(download_path) / float(1024 * 1024)
if round(fsize, 2) > max_file_size:
return {
'status':
400,
'code':
'InvalidImageSize',
'message':
'The input file size is too large (>%f MB)' %
float(max_file_size)
}
# 2.2.step 检查文件格式
# 图片格式检测（图片文件后缀可能不对）
download_path = os.path.normpath(download_path)
file_type = imghdr.what(download_path)
if file_type in ['jpeg', 'png', 'gif', 'bmp']:
# 图像文件
file_name = '%s.%s' % (data['file_name'], file_type)
os.rename(
download_path,
os.path.join('/'.join(download_path.split('/')[0:-1]),
file_name))
download_path = os.path.join(
'/'.join(download_path.split('/')[0:-1]), file_name)
else:
# 非图像文件
file_type = download_path.split('.')[-1]
if 'file_type' in self.demo_constraint:
if file_type not in self.demo_constraint['file_type']:
return {
'status':
400,
'code':
'InvalidImageFormat',
'message':
'The input file is not in a valid image format that the service can support'
}
return {'status': 200, 'path': download_path}
except:
print('Fail to download %s' % data['url'])
return {'status': 500, 'code': 'UnkownError'}
elif preprocess_type == 'RECEIVE':
try:
# 1.step 保存文件
file_path = data['file_path']
file_data = data['file_data']
with open(file_path, 'wb') as fp:
fp.write(file_data)
# 2.step 检查
# 2.1.step 检查文件大小
if 'file_size' in self.demo_constraint:
max_file_size = self.demo_constraint['file_size']
fsize = os.path.getsize(file_path) / float(1024 * 1024)
if round(fsize, 2) > max_file_size:
return {
'status':
400,
'code':
'InvalidImageSize',
'message':
'The input file size is too large (>%f MB)' %
float(max_file_size)
}
# 2.2.step 检查文件格式
# 图片格式检测（图片文件后缀可能不对）
file_path = os.path.normpath(file_path)
file_type = imghdr.what(file_path)
if file_type in ['jpeg', 'png', 'gif', 'bmp']:
# 图像文件
file_name = '%s.%s' % (
file_path.split('/')[-1].split('.')[0], file_type)
os.rename(
file_path,
os.path.join('/'.join(file_path.split('/')[0:-1]),
file_name))
file_path = os.path.join(
'/'.join(file_path.split('/')[0:-1]), file_name)
else:
# 非图像文件，通过获取扩展名作为文件类型
file_type = file_path.split('.')[-1]
if 'file_type' in self.demo_constraint:
if file_type not in self.demo_constraint['file_type']:
return {
'status':
400,
'code':
'InvalidImageFormat',
'message':
'The input file is not in a valid image format that the service can support'
}
return {'status': 200, 'path': file_path}
except:
return {'status': 500, 'code': 'UnkownError'}
elif preprocess_type == 'PACKAGE':
data_path = os.path.normpath(data['path'])
data_name = data_path.split('/')[-1]
ext_name = data_path.split('/')[-1].split('.')[-1].lower()
# 根据后缀判断上传的数据类型
if ext_name in ['jpg', 'jpeg', 'png', 'bmp', 'gif']:
return {'data': (data_path, data_name, 'IMAGE'), 'status': 200}
elif ext_name in ['mp4', 'avi', 'mov']:
return {'data': (data_path, data_name, 'VIDEO'), 'status': 200}
elif ext_name in ['txt']:
return {'data': (data_path, data_name, 'FILE'), 'status': 200}
else:
# TODO: support video and sound
logger.warn('dont support file type %s' % ext_name)
return {
'status': 400,
'code': 'InvalidPackage',
'message': 'Fail package'
}
elif preprocess_type == 'API_QUERY':
# 1.step base64解码
# format {'image': '', 'video': None, 'params': [{'data': ,'type': , 'name': ,},{}]}
image_str = None
if 'image' in data:
image_str = data['image']
if image_str is None:
return {
'status': 400,
'code': 'InvalidData',
'message': 'Missing query data'
}
image_b = base64.b64decode(image_str)
return {
'status': 200,
'data': {
'image': image_b,
'params': data['params']
}
}
def process_del_command(self):
dataset_name = FLAGS.dataset_name()
task_name = FLAGS.task_name()
experiment_name = FLAGS.experiment_name()
task_type = FLAGS.task_type()
task_measure = FLAGS.task_measure()
group_name = FLAGS.group()
user_name = FLAGS.user()
if group_name is not None and user_name is not None:
# delete user from group
users_info = {'users': [user_name]}
res = self.remote_api_request('hub/api/terminal/groups/%s'%str(group_name),
json.dumps(users_info),
'delete')
if res is not None:
logger.info('success to del user %s from group %s'%(str(user_name),str(group_name)))
else:
logger.error('fail to del user %s from group %s'%(str(user_name),str(group_name)))
return
if group_name is not None and user_name is None:
# detele group
logger.error('dont support del group')
return
if user_name is not None:
logger.warn('ignore user_name set')
if dataset_name is None and \
task_name is None and \
experiment_name is None and \
task_type is None:
logger.error('must set delete object [%s]' % ','.join(['dataset',
'task',
'experiment',
'task-type']))
return
if dataset_name is not None:
delete_remote_api = 'hub/api/terminal/delete/dataset'
response = self.remote_api_request(delete_remote_api,
action='delete',
data={'dataset-name': dataset_name})
if response['status'] != 'OK':
logger.error('delete error')
return
elif task_name is not None:
delete_remote_api = 'hub/api/terminal/delete/task'
response = self.remote_api_request(delete_remote_api,
action='delete',
data={'task-name': task_name})
if response['status'] != 'OK':
logger.error('delete error')
return
elif experiment_name is not None:
delete_remote_api = 'hub/api/terminal/delete/experiment'
response = self.remote_api_request(delete_remote_api,
action='delete',
data={'experiment-name': experiment_name})
if response['status'] != 'OK':
logger.error('delete error, maybe experiment name not unique')
return
elif task_type is not None:
data = {}
if task_measure is not None:
task_measures = task_measure.split(',')
task_measures = json.dumps(task_measures)
data['task-measures'] = task_measures
remote_api = 'hub/api/terminal/task/type/%s'%task_type
response = self.remote_api_request(remote_api,
action='delete',
data=data)
if response is None:
logger.error('fail to delete task type')
return
logger.info('success to delete task type')
print(response)