def render(args: argparse.Namespace) -> int:
kbutil_dll = get_kbutil_dll_path()
output_path = "case"
for piece in ["left", "center", "right"]:
input_path = "layout/triad_{}.xml".format(piece)
ret = generate_svg(kbutil_dll, input_path, output_path)
if ret != 0:
print("Failed to generate SVG")
return ret
bearings_path = "./temp/bearings_{}.json".format(piece)
bearings_debug_path = "./temp/bearings_{}.svg".format(piece)
mkdir(os.path.dirname(bearings_path))
ret = generate_bearings(
kbutil_dll, input_path, bearings_path, bearings_debug_path
)
if ret != 0:
print("Failed to switch bearings")
return ret
if args.open:
sh(
[
"inkscape",
os.path.join(output_path, "triad_left.svg"),
os.path.join(output_path, "triad_right.svg"),
os.path.join(output_path, "triad_center.svg"),
]
)
return 0
def gen_lang_site(master, site, config):
lang = site.lang
out_dir = "%s/%s" % (config.out_dir, lang)
util.mkdir(out_dir)
util.mkdir("%s/weeks" % out_dir)
util.mkdir("%s/days" % out_dir)
pages = standard_pages(site, config)
if not config.index_only:
pages += week_pages(site, config)
pages += day_pages(site, config)
if site.topics:
util.mkdir("%s/topics" % out_dir)
pages += topic_pages(site, config)
if site.facebook:
pages += facebook_pages(site, config)
if site.interviews:
pages += interviews_pages(site, config)
if site.custom:
pages += custom_pages(site, config)
if site.blogs:
util.mkdir("%s/blogs" % out_dir)
pages += blogs(site, config)
if site.gen_sitemap:
pages += [sitemap_page(master, site, config)]
if site.gen_search:
pages += [search_page(master, site, config)]
for page in pages:
with open_out_file(out_dir, page[0]) as outfile:
outfile.write(page[1])
print("Generated %s" % outfile.name)
def check_dirs(self):
mkdir(self.conf['dirs']['output'])
for key in self.conf['dirs']:
val = self.conf['dirs'][key]
if os.path.isdir(val) is False:
print('\nFolder does not exist ' + val + '\nExit.\n')
sys.exit(1)
def get_model_from_hdfs(self,model_path,feature_path,model_name):
dst_path=os.path.join(self.data_path,model_name)
util.mkdir(dst_path,self.logger)
rc = util.hdfs_file_copy(feature_path,dst_path,False,"hadoop",self.logger)
if(rc is False):
self.logger.error("Fail to get %s from hdfs",feature_path)
return False
rc = util.hdfs_file_copy(model_path,dst_path,False,"hadoop",self.logger)
if(rc is False):
self.logger.error("Fail to get %s from hdfs",model_path)
return False
return True
def __init__(self, type):
self.path = os.path.dirname(os.path.abspath(__file__))
self.logger = mylogging.getLogger('deploy_client.log')
self.version=time.strftime('%Y%m%d%H%M%S',time.localtime(time.time()))
self.online_path=os.path.join(self.path,'./online')
self.backup_path=os.path.join(self.path,'./backup')
self.data_path=os.path.join(self.path,'./data')
self.service_conf_path=os.path.join(self.path,'./config')
self.release_lib_path=os.path.join(self.path,'./release')
#to make sure data dir exsists ,and empty
if(type !='test'):
if os.access(self.data_path, os.F_OK):
util.del_file(self.data_path,self.logger)
else:
util.mkdir(self.data_path,self.logger)
if os.access(self.service_conf_path, os.F_OK):
util.del_file(self.service_conf_path,self.logger)
else:
util.mkdir(self.service_conf_path,self.logger)
if os.access(self.release_lib_path, os.F_OK):
util.del_file(self.release_lib_path,self.logger)
else:
util.mkdir(self.release_lib_path,self.logger)
util.mkdir(self.backup_path,self.logger)
util.mkdir(self.online_path,self.logger)
#read config file
config = ConfigParser.ConfigParser()
config.read('env.conf')
self.monitor=config.get('config', 'monitor');
self.model_name=config.get('config', 'model_name')
self.service_name=config.get('config', 'service_name')
self.model_path=config.get('config', 'model_path')
self.feature_path=config.get('config', 'feature_path')
self.model_port=config.get('config', 'model_port')
zk_regist=config.get('config', 'zk_regist')
zknodes=config.get('config', 'zklist').split(',')
for item in zknodes:
cluster_id=config.get(item, 'cluster_id')
group_id=config.get(item, 'group_id')
service_id=config.get(item, 'service_id')
type_id=config.get(item, 'type')
zkinfo={}
zkinfo['service_name'] = item
zkinfo['cluster_id'] = cluster_id
zkinfo['group_id'] = group_id
zkinfo['service_id'] = service_id
zkinfo['type_id'] = type_id
self.zklist.append(zkinfo)
if(zk_regist=='1'):
self.registe_zk_service
self.logger.info("regist zk %s",zkinfo)
def pool_gen_global_site_channels(pool, master):
config = master.config
site = master.global_site
pages = []
out_dir = "%s/global" % (config.out_dir)
results = []
if site.gen_channel_html and config.channel:
util.mkdir("%s/channels" % out_dir)
chunks = util.chunks(site.groups, 10)
for chunk in chunks:
results.append(
pool.apply_async(single_channel_pages, (site, config, chunk)))
return results
def _create_fs(query, router, edgenode):
from fuse import FUSE
from fs.vdfs import VDFS
if edgenode:
from lib.util import set_edgenode
set_edgenode()
mnt = get_mnt_path()
_unmount(mnt)
mkdir(mnt)
conf = get_conf_path()
mkdir(conf)
FUSE(VDFS(query, router), mnt, foreground=True)
def __init__(self, router=None, core=None):
path = get_var_path()
mkdir(path)
name = self._get_name()
if name in FIELDS:
self._field = name
else:
log_err(self, 'invalid entry')
raise Exception('Error: invalid entry')
if not router or router.local:
from interface.localfs import LocalFS
self._fs = LocalFS()
else:
from interface.remotefs import RemoteFS
self._fs = RemoteFS(router)
self._core = core
def get_model_from_hdfs(self,model_path,feature_path,model_name):
dst_path=os.path.join(self.data_path,model_name)
util.mkdir(dst_path,self.logger)
rc = util.hdfs_file_copy(feature_path,dst_path,False,"hadoop",self.logger)
if rc is False:
self.logger.error("Fail to get %s from hdfs",feature_path)
return False
local_models_path = os.path.join(self.path, "models")
if not os.path.isdir(local_models_path):
util.mkdir(local_models_path,self.logger)
rc = util.hdfs_file_copy(model_path,os.path.join(local_models_path, model_name), False,"hadoop",self.logger)
if rc is False:
self.logger.error("Fail to get %s from hdfs",model_path)
return False
predict_model_path = self.assemble_model_file(os.path.join(local_models_path,model_name))
print_log("assemble predict model path:%s\n" % predict_model_path)
rc = util.copy_file(predict_model_path,os.path.join(dst_path,"predict.model"),self.logger)
if rc is False:
print_log("copy predict model to data dir error:%s\n")
return False
return True
def packet_dnn(self):
dst_path=os.path.join(self.data_path, self.model_name, "version")
f_path = os.path.join(self.data_path, self.model_name)
util.mkdir(dst_path,self.logger)
print_log("client begin packeting model\n")
#get model
rc = util.hdfs_file_copy(self.model_path,dst_path,False,"hadoop",self.logger)
if rc is False:
print_log("Fail to get %s from hdfs" % self.model_path)
return False
rc = util.hdfs_file_copy(self.feature_path,f_path,False,"hadoop",self.logger)
if rc is False:
print_log("Fail to get %s from hdfs" % self.feature_path)
return False
#modify config
rc = self.modify_config(self.model_name)
if rc is False:
return False,'modify config error'
#get release so lib
self.get_release_so()
online_path=os.path.join(self.online_path,self.service_name,self.version);
tar_path=os.path.join(online_path,self.version)
shutil.move(self.service_conf_path,os.path.join(tar_path,"./config"))
shutil.move(self.data_path,os.path.join(tar_path,"./data"))
shutil.move(self.release_lib_path,os.path.join(tar_path,"./lib"))
tar_file = util.tar(tar_path,self.logger)
util.del_dir(tar_path,self.logger)
#res = util.get_file_md5(tar_file)
res=util.gen_file_md5(online_path,tar_file,self.version,self.logger)
#upload to monitor
cmd = "rsync -r %s %s" %(os.path.join(self.online_path,self.service_name),self.monitor)
print_log("packet upload cmd: %s\n" % cmd)
rc,stdout,stderr= util.run_shell(cmd,self.logger)
if rc is False:
return False,'upload model error'
self.logger.info("success upload model %s to monitor %s,model version %s",self.model_name,self.monitor,self.version)
print_log("client finish packet: %s\n" % tar_file)
return True, self.version
def ponoko(args: argparse.Namespace) -> int:
kbutil_dll = get_kbutil_dll_path()
output_path = os.path.join(get_triad_repo_dir(), "case", "ponoko")
mkdir(output_path)
for piece in ["left", "center", "right"]:
print(piece)
input_path = "layout/triad_{}.xml".format(piece)
ret = generate_svg(kbutil_dll, input_path, output_path)
if ret != 0:
print("Failed to generate SVG")
return ret
if args.open:
sh([
"inkscape",
os.path.join(output_path, "triad_left.svg"),
os.path.join(output_path, "triad_right.svg"),
os.path.join(output_path, "triad_center.svg"),
])
return 0
def packageKSimhash(requir_user_file, thsold):
''' 对prov_sub数据通过user作为质点，进行降维
@param requir_user_file 原始数据文件(user_id, province， question)
@param thsold 阈值，如果小于此数值即为有用数值，int
'''
prov_set = getProvinceSet()
with open(USER_PATH + requir_user_file, 'r') as user_file:
readers = csv.DictReader(user_file)
for reader in readers:
prov = reader['province'][:2]
if prov not in prov_set:
prov = '全国'
user_id = reader['user_id']
question_id = eval(reader['question'])
question_list = tableToJson(table='question_simhash_20171111',
question_id=question_id)
if len(question_list) > 0:
subj_dic = {}
recom_set = set([])
for sub_kpoint in question_list:
if str(sub_kpoint[1]) not in subj_dic.keys():
subj_dic[str(sub_kpoint[1])] = [sub_kpoint[0]]
else:
subj_dic[str(sub_kpoint[1])].append(sub_kpoint[0])
for ks, vs in subj_dic.items():
ps_file = PROV_SUB_PATH + datetime + '/' + prov + '/' + prov_subj_file.format(
prov, ks, datetime)
if os.path.exists(ps_file):
PATH = K_SIMH_PATH + datetime + '/' + prov
mkdir(PATH)
if os.path.exists(
PATH + '/' +
output_file.format(prov, ks, datetime)):
os.remove(PATH + '/' +
output_file.format(prov, ks, datetime))
k_simh_list = []
with open(ps_file, 'r') as ksimhash_file:
while True:
ksimhash = ksimhash_file.readline()
if ksimhash:
sim_dis = K_Simhash(
ksimhash=eval(ksimhash), vs=vs)
if sim_dis < thsold:
with open(
PATH + '/' +
output_file.format(
prov, ks, datetime),
'a') as txt_file:
txt_file.writelines(
json.dumps(eval(ksimhash)))
txt_file.write('\n')
else:
break
ksimhash_file.close()
else:
print(u"没有在Prov_Sub_input中查询到生成的（省份-学科）文件！")
_DIR = os.path.dirname(
os.path.dirname(os.path.dirname(os.path.abspath(__file__)))).replace(
'\\', '/')
DATABASE = _DIR + '/new_database/'
RAW_PATH = DATABASE + 'Raw_input/'
SUB_KPOINT_PATH = DATABASE + 'Sub_kpoint_input/'
datetime = '09-11'
init_file = 'user_if_{}.csv'
skp_file = 'question_sub_kpoint_{}_{}.txt'
if __name__ == '__main__':
province_set = {
'nan', '0', '1', '2', '3', 'Rize', 'Juzny', 'hobbit', '全国', '台湾',
'NULL'
}
mkdir(SUB_KPOINT_PATH + datetime)
PATH = SUB_KPOINT_PATH + datetime + '/'
with open(RAW_PATH + init_file.format(datetime), 'r',
encoding='utf-8') as raw_file:
readers = csv.DictReader(raw_file)
for reader in readers:
prov = reader['province']
if prov in province_set:
prov = '全国'
else:
prov = prov[:2]
question_id = []
try:
if isinstance(reader['question'], str):
if len(eval(reader['question'])) > 0:
else:
break
logging.info(u"已经解析完{0}省份下{1}学科的数据，并存入到Prov_Sub_input文件！".format(prov, subj))
new_file.close()
sub_kpoint_file.close()
if __name__ == '__main__':
prov_set = getProvinceSet()
# prov_set = {'青海'}
subj_set = {str(j) for j in range(1, 11)} | {str(j) for j in range(21, 31)} | {str(j) for j in range(41, 51)}
pool = Pool(3)
for prov in prov_set:
P_S_PATH = PROV_SUB_PATH + datetime + '/' + prov
mkdir(P_S_PATH)
logging.info("the classify the subject ")
for subj in subj_set:
logging.info(u"正在读取{0}省份下{1}学科的Sub_kpoint_input文件".format(prov, subj))
pool.apply_async(packClassSub, kwds={
"prov":prov,
"subj":subj,
"P_S_PATH":P_S_PATH
})
pool.close()
pool.join()
subj_set = {str(j)
for j in range(1, 11)} | {str(j)
for j in range(21, 31)
} | {str(j)
for j in range(41, 51)}
for datetime in datetimes:
logging.basicConfig(
format=LOGGING_FORMAT,
level=logging.INFO,
filename='working/Recom_subqid_{}.log'.format(datetime),
filemode='a')
for prov in prov_set:
PATH = SUBJ_KPO_PATH + datetime + '/' + prov
mkdir(PATH)
logging.info("running the {0} at the time between {1}".format(
prov, datetime))
for subj in subj_set:
if os.path.exists(PATH + '/' +
output_file.format(prov, subj, datetime)):
os.remove(PATH + '/' +
output_file.format(prov, subj, datetime))
pool.apply_async(packSubjKpo,
kwds={
"prov": prov,
"PATH": PATH,
"datetime": datetime
})
total_num = len(X)//slice_num # map to nodes
if slidx != slice_num-1:
X = X[slidx*total_num: (slidx+1)*total_num]
else:
X = X[slidx*total_num: ]
total_num = len(X)
total_idx = np.arange(total_num)
chunk_len = total_num//(mpisize-1) if total_num >= mpisize - 1 else total_num # map to process
data = [X[total_idx[i*chunk_len: (i+1)*chunk_len]] for i in tqdm(range(mpisize), desc='Scatter data')] # scatter the data to other process
else:
data = None
data = comm.scatter(data, root=0)
# TMF
data_path = '../feature-{3}/{0}/{1}/{2}/'.format(*[mode, freq, slidx, model_type])
mkdir(data_path)
all_feature = []
win = 3000 # length of the time series
D = 3 # order of the motif, 3 means triad
shape = np.array([len(range(1, (win-1)//(D-1) + 1)), len(range(0, win-(D-1)*1)), D]) # TMF image shape
overlap = win-(D-1)*shape[0] # overlap cause by the rotation in the TMF image
TMF = np.zeros(shape) # placeholder of TMF image
for ts in tqdm(data):
filt_ts = filt_ECG(ts, freq)
TMF = gen_TMF(filt_ts, overlap, TMF, D) # the gen_TMF function is optimized with numba package
img = np.expand_dims(TMF, axis=0) # [1, W, H, 3]
feature = extractor.predict(img) # [1, 512]
all_feature.append(feature)
def __init__(self, action, model_name, service_name, algorithm):
print_log("model name: " + model_name + ",service name: " + service_name)
self.path = os.path.dirname(os.path.abspath(__file__))
self.restore_env_conf()
self.algorithm = algorithm
self.logger = mylogging.getLogger('deploy_client.log')
cmd = "rm -rf " + os.path.join(self.path, "models/*")
rc, stdout, stderr = util.run_shell(cmd,self.logger)
self.version=time.strftime('%Y%m%d%H%M%S',time.localtime(time.time()))
self.online_path=os.path.join(self.path,'./online')
self.backup_path=os.path.join(self.path,'./backup')
self.data_path=os.path.join(self.path,'./data')
self.service_conf_path=os.path.join(self.path,'./config')
self.release_lib_path=os.path.join(self.path,'./release')
self.docker_images['LR'] = "registry.intra.weibo.com/weibo_rd_algorithmplatform/modelservice_lr:v1.1"
self.docker_images['DNN'] = "registry.intra.weibo.com/weibo_rd_algorithmplatform/dnn_prod:v1.0"
self.modify_zk_info(model_name, service_name)
#to make sure data dir exsists ,and empty
if os.access(self.data_path, os.F_OK):
util.del_file(self.data_path,self.logger)
else:
util.mkdir(self.data_path,self.logger)
if os.access(self.service_conf_path, os.F_OK):
util.del_file(self.service_conf_path,self.logger)
else:
util.mkdir(self.service_conf_path,self.logger)
if os.access(self.release_lib_path, os.F_OK):
util.del_file(self.release_lib_path,self.logger)
else:
util.mkdir(self.release_lib_path,self.logger)
util.mkdir(self.backup_path,self.logger)
util.mkdir(self.online_path,self.logger)
#read config file
config = ConfigParser.ConfigParser()
config.read('env.conf')
self.monitor=config.get('config', 'monitor');
self.docker_image_tag=self.docker_images[algorithm.upper()]
print_log("docker tag: %s" % self.docker_image_tag)
self.model_name=model_name
self.service_name=service_name
self.get_model_feature_path(action)
self.model_port=config.get('config', 'model_port')
zk_regist=config.get('config', 'zk_regist')
zknodes=config.get('config', 'zklist').split(',')
for item in zknodes:
cluster_id=config.get(item, 'cluster_id')
group_id=config.get(item, 'group_id')
service_id=config.get(item, 'service_id')
type_id=config.get(item, 'type')
zkinfo={}
zkinfo['service_name'] = item
zkinfo['cluster_id'] = cluster_id
zkinfo['group_id'] = group_id
zkinfo['service_id'] = service_id
zkinfo['type_id'] = type_id
self.zklist.append(zkinfo)
if zk_regist=="1" and action =="deploy":
self.registe_zk_service()
self.logger.info("regist zk %s",zkinfo)
prov_set = getProvinceSet()
subj_set = {str(j)
for j in range(1, 11)} | {str(j)
for j in range(21, 31)
} | {str(j)
for j in range(41, 51)}
# subj_set = {'2'}
# prov_set = {'福建'}
for datetime in datetimes:
logging.basicConfig(
format=LOGGING_FORMAT,
level=logging.INFO,
filename='working/itemCF_Eval_{}.log'.format(datetime),
filemode='a')
for prov in prov_set:
ICF_PATH = ITEMCF_PATH + datetime + '/' + prov
mkdir(ICF_PATH)
for subj in subj_set:
pool.apply_async(packItemCFEval,
kwds={
'prov': prov,
'subj': subj,
'datetime': datetime,
'ICF_PATH': ICF_PATH
})
pool.close()
pool.join()
ps_file.close()
logging.info(u"已经解析完{0}省份下{1}学科的数据，并存入到文件！".format(prov, subj))
if __name__ == '__main__':
prov_set = getProvinceSet()
subj_set = {str(j)
for j in range(1, 11)} | {str(j)
for j in range(21, 31)
} | {str(j)
for j in range(41, 51)}
# prov_set = {'全国'}
# subj_set = {'3'}
pool = Pool(2)
for prov in prov_set:
FO_PATH = FPGROWTH_PATH + datetime + '/' + prov
mkdir(FO_PATH)
for subj in subj_set:
pool.apply_async(packageFPGrowthRun,
kwds={
"prov": prov,
"subj": subj,
"datetime": datetime,
"FO_PATH": FO_PATH
})
pool.close()
pool.join()
support = support * 0.9
confidence = confidence * 0.9
flag += 1
if __name__ == '__main__':
#prov_set = getProvinceSet()
# , '甘肃', '宁夏', '四川', '全国', '重庆', '陕西', '吉林', '北京', '海南'
prov_set = {'江苏'}
support = 0.03
confidence = 0.10
# subj_set = {str(j) for j in range(1, 11)} | {str(j) for j in range(21, 31)} | {str(j) for j in range(41, 51)}
subj_set = {'5'}
for prov in prov_set:
AO_PATH = APRIORI_PATH + datetime + '/' + prov
mkdir(AO_PATH)
for subj in subj_set:
packageWhileAprioriRun(prov=prov,
subj=subj,
support=support,
confidence=confidence,
datetime=datetime,
AO_PATH=AO_PATH)
# pool = Pool()
# for prov in prov_set:
# AO_PATH = APRIORI_PATH + datetime + '/' + prov
# mkdir(AO_PATH)
# for subj in subj_set:
# pool.apply_async(packageAprioriRun, kwds={
""" Map the parallel_flex.py to different nodes and cores to run them parallelly. Author: Yadong Zhang E-mail: [email protected] Demo: $ python3 map_length_effect.py 10 """ import sys sys.path.append('../') import subprocess from lib.util import mkdir nodes = int(sys.argv[1]) for idx in range(nodes): mkdir('slice/{0}/'.format(idx)) qsub_command = """qsub -v slice={0},nodes={1} -q adf length_effect.sh""".format( *[idx, nodes]) exit_status = subprocess.call(qsub_command, shell=True) # upload if exit_status is 1: # Check to make sure the job submitted print("Job {0} failed to submit".format(qsub_command))
os.path.dirname(os.path.dirname(os.path.abspath(__file__)))).replace(
'\\', '/')
DATABASE = _DIR + '/new_database/Input/'
RAW_PATH = DATABASE + 'Raw_input/'
PRE_DATA_PATH = DATABASE + 'Pre_data_input/'
init_file = 'user_if_{}.csv'
pre_province_file = 'user_json_{}_{}.csv'
pre_province_txt = 'user_json_{}_{}.txt'
datetime = '09-11'
if __name__ == '__main__':
province_set = {
'nan', '0', '1', '2', '3', 'Rize', 'Juzny', 'hobbit', '全国', '台湾',
'NULL'
}
mkdir(PRE_DATA_PATH + datetime)
PATH = PRE_DATA_PATH + datetime + '/'
with open(RAW_PATH + init_file.format(datetime), 'r',
encoding='utf-8') as raw_file:
# data = csv.reader(f)
# for d in data:
# print(d)
readers = csv.DictReader(raw_file)
for reader in readers:
prov = reader['province']
if prov in province_set:
prov = '全国'
elif prov in {'闽'}:
prov = '福建'
else:
prov = prov[:2]
