def check(self, work_copy, address, user, password, log=None):
baseVer = self.get_work_ver(work_copy, user, password, log)
headVer = self.get_rep_ver(address, user, password, log)
if headVer == baseVer:
FileUtil.log(log, work_copy + ": No SVN change found!")
return False
return True
def get_rep_ver(self, rep, user, password, log=None):
(headOutput,errno) = process.get_run_output("svn log -q -l 1 " + StrUtil.quote(rep) + self._get_svn_para(user, password))
FileUtil.log(log, headOutput)
if errno != 0:
raise Exception("get version of " + rep + " error")
#注意这地方要处理中英文版本，所以要识别GB码的‘版本’和‘ Revision’
headVer = re.search("r([0-9]+) \|", headOutput).group(1)
return headVer
def get_work_ver(self, work_copy, user, password, log=None):
(baseOutput,errno) = process.get_run_output("svn info -r BASE " + StrUtil.quote(work_copy) + self._get_svn_para(user, password))
FileUtil.log(log, baseOutput)
if errno != 0:
raise Exception("get version of " + work_copy + " error")
if re.search("Last Changed Rev: ([0-9]+)", baseOutput) is None:
baseVer = re.search("\xd7\xee\xba\xf3\xd0\xde\xb8\xc4\xb5\xc4\xb0\xe6\xb1\xbe: ([0-9]+)", baseOutput).group(1)
else:
baseVer = re.search("Last Changed Rev: ([0-9]+)", baseOutput).group(1)
return baseVer
def create_file_input(self, message):
fileUtil = FileUtil()
lst_char = message[:]
all_message = ''
for j in range(len(lst_char)):
if j == 0:
ans = 'B'
else:
ans = 'I'
message_model = lst_char[j]+' '+self.get_group(lst_char[j])+' '+ans+'\n'
all_message = all_message+message_model
fileUtil.write_newfile('crf.test.data', all_message)
def on_rename_file(self, command):
originPath = command["originPath"]
renamePath = command["renamePath"]
success = True
message = "成功！"
try:
# 复制文件到指定目录
FileUtil.rename_file(originPath, renamePath)
except Exception as e:
success = False
message = str(e)
finally:
return {"success": success, "message": message}
def make(self, projectConfig):
makeConf = {
'target_type':
projectConfig.getItem('make.configuration.target.type',
self.defaultConf['target_type']),
'ar':
projectConfig.getItem('make.configuration.archiver.ar',
self.defaultConf['ar']),
'ld':
projectConfig.getItem('make.configuration.linker.ld',
self.defaultConf['ld']),
'ldflags':
projectConfig.getItem('make.configuration.linker.flags',
self.defaultConf['ldflags']),
'ld_library_paths':
projectConfig.getItem('make.configuration.linker.library_paths',
self.defaultConf['ld_library_paths']),
'libraries':
projectConfig.getItem('make.configuration.linker.libraries', []),
'solution_dir':
projectConfig.getItem('global.solution.dir', '..'),
'sources':
projectConfig.getItem('make.configuration.linker.include', []),
}
makefile = GMakeDocument(TARGET_MAKEFILE_PATH)
objectSourcePaths = []
self.objectFiles.clear()
solutionDirPath = Path(makeConf['solution_dir'], True)
objectFilePaths = []
for objectSourcePath in makeConf['sources']:
objectSourcePath = solutionDirPath.join(objectSourcePath)
FileUtil.searchAllFiles(objectFilePaths, objectSourcePath, ['.o'])
for objectFilePath in objectFilePaths:
self.objectFiles.append(Path(objectFilePath))
finalFilePath = self.getFinalFilePath(projectConfig, makeConf)
linkRule = self.getLinkRule(finalFilePath, makeConf)
subMakefile = GMakeDocument(FINAL_TARGET_SUBMAKE_PATH)
subMakefile.addRule(linkRule)
makefile.addSubDocument(subMakefile)
allRule = GMakeSimpleRule('all', [finalFilePath])
makefile.addRule(allRule)
makefile.writeToFile()
return True
def process_item(self, item, spider):
image_urls = []
for image_url in item['image_urls']:
url = image_url.get('url')
urlHash = EncryptUtil.md5(url)
path = 'full/' + str(urlHash) + '.jpg'
detailPath = self.savePath + '/' + path
# 创建目录
saveDir = self.savePath + '/full'
if not FileUtil.dirIsExist(saveDir):
FileUtil.createDir(saveDir)
if FileUtil.fileIsExist(detailPath):
spider.logDao.info(u'图片已经存在本地:' + url)
image_url_new = {'ok': True, 'x': {'url': url, 'path': path}}
else:
try:
fileResponse = requests.get(url, timeout=10)
req_code = fileResponse.status_code
req_msg = fileResponse.reason
if req_code == 200:
open(detailPath, 'wb').write(fileResponse.content)
image_url_new = {
'ok': True,
'x': {
'url': url,
'path': path
}
}
spider.logDao.info(u'图片成功下载:' + url)
else:
spider.logDao.info(u'下载图片失败:' + url)
image_url_new = {
'ok': False,
'x': {
'url': url,
}
}
except Exception, e:
print e
spider.logDao.warn(u'下载图片失败:' + url)
image_url_new = {
'ok': False,
'x': {
'url': url,
}
}
image_urls.append(image_url_new)
# 空转2s
TimerUtil.sleep(2)
def downloadFromUrlAndSaveToFile(url, fileName, skipIfFileExists):
if not(skipIfFileExists or os.path.isfile(fileName)):
try:
u = urllib2.urlopen(url)
try:
FileUtil.ensureDirectoryForFile(fileName)
with open(fileName, "w") as f:
lines = u.readlines()
f.write(lines[0])
f.writelines(reversed(lines[1:]))
f.closed
finally:
u.close()
except urllib2.HTTPError: logging.warning("Cannot load from: " + url)
def on_copy_dir_to_dir(self, command):
originPath = command["originPath"]
destPath = command["destPath"]
success = True
message = "成功！"
try:
# 判断源文件夹是否存在
FileUtil.validate_folder_exists(originPath, False)
# 复制文件到指定目录
FileUtil.copy_folder(originPath, destPath)
except Exception as e:
success = False
message = str(e)
finally:
return {"success": success, "message": message}
def setEditByApkPath(self, filePath):
if not filePath:
return
filePath = QtCore.QDir.toNativeSeparators(filePath)
self.srcApkEdit.setText(filePath)
destPath = FileUtil.getFilePathWithName(filePath)
self.destApkEdit.setText(destPath)
def main(self, excle):
print '===============start all==================='
# 数据初始化
fileDao = FileUtil.FileDao()
# 单个excle处理
data = fileDao.open_excel(excle)
table = data.sheets()[0]
nrows = table.nrows # 行数
ncols = table.ncols # 列数
print ('行数：%s ,列数：%s' %(nrows , ncols))
# measure_unit_table = data.sheets()[1]
#
# for rownum in range(1, measure_unit_table.nrows):
# row = measure_unit_table.row_values(rownum)
# key = row[0].strip()
# value = row[1].strip()
# self.measure_unit_dic[key] = value
# 第一行
self.first_row_process(table , ncols)
# 第二行开始的后面所有
self.next_row_process(table, nrows, ncols)
print '===============end all==================='
def getApkApplicationIconPath(self, apkPath, apkInfo=None):
apkPath = unicode(apkPath)
applicationIconStr = self.getApkApplicationIconPathInApk(
apkPath, apkInfo)
if not applicationIconStr:
return u'未获取到应用图标'
apkZip = zipfile.ZipFile(apkPath)
iconData = apkZip.read(applicationIconStr)
iconDestPath = FileUtil.getFilePathWithName(apkPath)
FileUtil.mkdirNotExist(iconDestPath)
saveIconPath = os.path.join(iconDestPath, 'icon_launcher.png')
if FileUtil.isFileOrDirExist(saveIconPath):
return saveIconPath
with open(saveIconPath, 'w+b') as saveIcon:
saveIcon.write(iconData)
return saveIconPath
def __init__(self):
self.dao = CrawlDao.CrawlDao()
self.fileDao = FileUtil.FileDao()
self.stringUtil = StringUtil.StringUtil()
self.table_mapping = 'db_category_mapping'
self.vehicle_dict = {"商用车": "H", "乘用车": "C"}
# 新分类的
self.cate_dict = dict()
cate_sql = "select cat_id,cat_name,vehicle_code,cat_level,parent_id from db_category_new "
cate_array = self.dao.db.get_data(cate_sql)
for cate_data in cate_array:
cat_id = str(cate_data['cat_id'])
cat_name = str(cate_data['cat_name'])
cat_level = str(cate_data['cat_level'])
vehicle_code = str(cate_data['vehicle_code'])
parent_id = str(cate_data['parent_id'])
key = cat_name + "_" + cat_level + "_" + parent_id
if cat_level == '3':
key += "_" + vehicle_code
self.cate_dict[key] = cat_id
# 老cate的图片数据
self.old_cat_dict = dict()
old_cat_sql = "select cat_name,category_thumb from db_category where cat_id < 2999 and parent_id = 0 and is_deleted = 'N'"
old_cate_array = self.dao.db.get_data(old_cat_sql)
for old_cate_data in old_cate_array:
cat_name = str(old_cate_data['cat_name'])
category_thumb = str(old_cate_data['category_thumb'])
self.old_cat_dict[cat_name] = category_thumb
def load_data(self):
"""
1. 读取原始数据。ratings数据格式为[Rating]
"""
ratings = FileUtil.readRatings(fileName=self.DataPath)
# 划分训练集和测试集
train, test = ratioSplitter(ratings, self.Ratio, fixed=True)
# 获取user和item列表,这里需要去重，并保持顺序
users = np.unique([r.uid for r in train])
items = np.unique([r.iid for r in train])
self.logger.debug(f"users长度:{len(users)}")
self.logger.debug(f"items长度:{len(items)}")
# 构建训练集评分矩阵
R_train = pd.DataFrame(np.zeros((len(users), len(items))), index=users, columns=items)
for r in train:
R_train.at[r.uid, r.iid] = 1 # 隐式反馈，这里就设置为1，忽略具体的评分数据
self.logger.debug(f"训练集评分矩阵:\n{R_train}")
# 构建测试集评分矩阵
test_users = np.unique([r.uid for r in test])
test_items = np.unique([r.iid for r in test])
R_test = pd.DataFrame(np.zeros((len(test_users), len(test_items))), index=test_users, columns=test_items)
for r in test:
R_test.at[r.uid, r.iid] = 1 # 隐式反馈，这里就设置为1，忽略具体的评分数据
self.logger.debug(f"测试集评分矩阵:\n{R_test}")
self.users = users
self.items = items
self.R_train = R_train
self.R_test = R_test
return users, items, R_train, R_test
def __init__(self):
self.dao = CrawlDao.CrawlDao("modeldatas")
self.fileDao = FileUtil.FileDao()
self.stringUtil = StringUtil.StringUtil()
self.sql_category_table = 'db_category'
self.sql_part_table = 'db_category_part'
# 已更新的分类 name+level+parent_id
self.category_have_list = list()
# 已更新的part name+level
self.part_have_list = list()
self.label_dict = {u'字标': '1', u'灯泡': '2', u'四滤': '3', u'': '0'}
self.cat_kind_dict = {u'全车件': 1, u'易损件': 0}
# 分类无法判断标识
self.cat_kind_not = '3'
self.aliss_name_text = ''
self.label_text = ''
# db_category level:dict(name+parent_id)
self.level_name = self.init_category()
# db_category_part name+cate_id:dict(part)
# self.part_name = self.init_part()
self.init_part()
# name+level+parent_id:id
self.category_id_dict = dict()
# name+level+parent_id:vehicle_code
self.category_code_dict = dict()
# cat_id:cat_name
self.cat_id_name = dict()
# 是否是新增的类目
self.is_new = False
def read_data(filename):
context = FileUtil.readFile(filename)
items = []
for j, c in enumerate(context):
questionTriple = c.split("\t")
question = tokenizer(questionTriple[-1].lower())
questionTriple[:-1] = [questionTriple[i].replace("www.freebase.com/", "").replace("/", ".") for i in
range(0, len(questionTriple) - 1)]
sub = questionTriple[0]
relation = rel_voc[questionTriple[1]]
obj = questionTriple[2]
if sub == 'm.07s9rl0':
sub = 'm.02822'
if obj == 'm.07s9rl0':
obj = 'm.02822'
if sub in entityId2type:
sub_type = entityId2type[sub]
else:
sub_type = []
if sub in enName:
sub_text = enName[sub]
else:
sub_text = ""
sub_text.split(" ")
if sub_text in question:
anonymous_question = get_anonymous(sub_text, question)
else:
anonymous_question = question
it = Item(j, question, sub, relation, obj, list(sub_type), sub_text, anonymous_question)
items.append(it)
return items
def getImage(imageId):
if not FileUtil.exists(join(settings.images_cache_folder, imageId)):
mapfileId = HbaseUtil.getMapFileId(imageId)
logging.info(mapfileId)
if mapfileId is None:
logging.error('404')
response.status = 404
return
if FileUtil.exists(join(settings.mapfile_cache_folder, mapfileId)):
MapFileUtil.readMapFile(mapfileId)
else:
MapFileUtil.readMapFileFromHdfs(mapfileId)
# folder_path = join(settings.images_cache_folder,mapfileId)
folder_path = settings.images_cache_folder
logging.info(folder_path)
return static_file(imageId, root=folder_path, mimetype='image/jpg')
def getTweets(self, name='random', searchterm=None):
name = name.lower()
if name == 'random':
name = random.choice(self.twitterUsernames.keys())
if name not in self.twitterUsernames:
return (False, "I don't know anybody by the name of '{}', sorry. ".format(name))
tweetFileName = os.path.join(GlobalStore.scriptfolder, 'data', 'tweets', '{}.txt'.format(self.twitterUsernames[name]))
if not os.path.exists(tweetFileName):
self.executeScheduledFunction()
return (False, "I don't seem to have the tweets for '{}', sorry! I'll retrieve them right away, try again in a bit".format(name))
tweets = FileUtil.getAllLinesFromFile(tweetFileName)
if searchterm is not None:
#Search terms provided! Go through all the tweets to find matches
regex = None
try:
regex = re.compile(searchterm, re.IGNORECASE)
except (re.error, SyntaxError):
self.logWarning("[STtip] '{}' is an invalid regular expression. Using it literally".format(searchterm))
for i in xrange(0, len(tweets)):
#Take a tweet from the start, and only put it back at the end if it matches the regex
tweet = tweets.pop(0)
if regex and regex.search(tweet) or searchterm in tweet:
tweets.append(tweet)
if len(tweets) == 0:
return (False, "Sorry, no tweets matching your search were found")
else:
return (True, tweets)
def MakeBuildVersion():
versionFile = None
if not PathUtil.exist('target/BUILD_VERSION'):
versionFile = FileUtil.openFile('target/BUILD_VERSION')
else:
versionFile = open('target/BUILD_VERSION', 'w')
versionFile.close()
def MakeBuildVersion():
versionFile = None
if not PathUtil.exist('target/BUILD_VERSION'):
versionFile = FileUtil.openFile('target/BUILD_VERSION')
else:
versionFile = open('target/BUILD_VERSION', 'w')
versionFile.close()
def __init__(self):
self.dao = CrawlDao.CrawlDao()
self.fileDao = FileUtil.FileDao()
self.stringUtil = StringUtil.StringUtil()
# p_id+code : id
self.cate_id_dict = dict()
# self.cate_name_dict = dict()
cate_sql = 'SELECT cat_id,cat_name,cat_code,parent_id,vehicle_code,cat_level FROM db_category_dian WHERE cat_id > 2999 '
cate_array = self.dao.db.get_data(cate_sql)
for cate_data in cate_array:
cat_id = str(cate_data['cat_id'])
cat_code = str(cate_data['cat_code'])
parent_id = str(cate_data['parent_id'])
cat_level = str(cate_data['cat_level'])
vehicle_code = str(cate_data['vehicle_code'])
key = str(parent_id) + "_" + str(cat_code)
if cat_level == '3':
key += "_" + vehicle_code
self.cate_id_dict[key] = cat_id
# self.cate_name_dict[cat_id] = str(cate_data['cat_name'])
# g_id:old_cat_id
self.goods_dict = dict()
goods_sql = "select cat_id,goods_id from db_goods where seller_id = 1"
goods_array = self.dao.db.get_data(goods_sql)
for goods_data in goods_array:
self.goods_dict[str(goods_data['goods_id'])] = str(
goods_data['cat_id'])
def test_save(model, config, tf_transition_params_test, dataset):
step = 1
ans = []
t = 0
if dataset == "test":
iters = model.testing_iters
else:
iters = model.deving_iters
tp = 0
fn = 0
fp = 0
model.qa.batch_index_dev = 0
model.qa.batch_index_test = 0
while (step - 1) * config.batch_size < iters:
x_lens, test_x, y = model.qa.load_data(config.batch_size, dataset)
t += len(x_lens)
tf_unary_scores = model.sess.run(model.unary_scores,
feed_dict={model.question_ids: test_x,
model.x_lens: x_lens})
for tf_unary_scores_, y_, sequence_lengths_t_ in zip(tf_unary_scores, y,
x_lens):
tf_unary_scores_ = tf_unary_scores_[:sequence_lengths_t_]
viterbi_sequence, _ = tf.contrib.crf.viterbi_decode(
tf_unary_scores_, tf_transition_params_test)
ans.append(viterbi_sequence)
for i, p in enumerate(viterbi_sequence):
if p == 1 and y_[i] == 1:
tp += 1
elif p == 0 and y_[i] == 1:
fn += 1
elif p == 1 and y_[i] == 0:
fp += 1
step += 1
pred = tp * 1.0 / (tp + fp)
recall = tp * 1.0 / (tp + fn)
f1 = 2 * pred * recall / (pred + recall)
print("{} pred: {} recall: {} f1: {}".format(dataset, pred, recall, f1))
questions = model.qa.get_questions(dataset)
print("ans len:\t", t)
print("question:\t", len(questions))
output = []
for i in range(len(questions)):
output.append("qid: {}".format(i))
output.append(FileUtil.list2str(label2entity(questions[i], ans[i]), split="\t"))
FileUtil.writeFile(output, "{}/sq.{}.label".format(config.answer_path, dataset))
def _transfer(self, size):
images = RedisUtil.pop(self.num, "prepare")
if images:
if RedisUtil.push(images, "image") and RedisUtil.addSize(size):
if FileUtil.moveImages(images, self.prepare_dir,
self.queue_dir):
return True
return False
def checkAllIpsByMultiThreading(fileName):
from util import FileUtil
ips = FileUtil.readIPsFromFile(fileName)
executor = ThreadPoolExecutor(NO_OF_PROCESS)
results = executor.map(getIPPingStatus, ips)
dataList = list(results)
for i in dataList:
print(i)
def checkAllIpsByMultiprocessing(fileName):
ips = FileUtil.readIPsFromFile(fileName)
pool = Pool(processes=NO_OF_PROCESS)
result = pool.map_async(getIPPingStatus, ips)
pool.close()
dataList = result.get()
for i in dataList:
print(i)
def run(self):
if self._createMapFile():
if self._writeToHDFS() and self._addRecord():
if FileUtil.removeFolder(self.mapFileId):
return True
else:
logging.error("fail to produce mapfile")
return False
def get_hist_data(start=None, end=None):
stocks = ts.get_stock_basics()
tmonths= dataUtil.getListMonth(start,end)
for tmonth in tmonths :
for stock in stocks.index :
stockInfo = ts.get_hist_data(code=stock, start=tmonth[1], end=tmonth[2])
if (not stockInfo is None and not len(stockInfo) == 0):
code = pd.DataFrame({"code": np.array([stock]*len(stockInfo))},index=stockInfo.index)
df = pd.concat([code,stockInfo],axis=1)
#dataPath = TushareApi.path + "hist_data/dt=" + str(tmonth[0]) + "/" + str(stock) + ".csv"
dataPath = TushareApi.path + "hist_data/dt=" + str(tmonth[0]) + "/stock.csv"
fileUtil.saveDf(df,dataPath,"a")
def loadRawData(self, path):
# logging.info("开始加载原始数据,path = %s", path)
# self.__timer.start()
self.__rawData = FileUtil.readAllLines(path)
self.__rawData.pop(0) # 删除第一行的title
# logging.debug("原始数据长度:%d,原始数据:%s", len(self.__rawData), str(self.__rawData))
# logging.info('加载数据完成,用时:%dms', self.__timer.stop())
return self.__rawData
pass
def writeToFile(self):
makefileContent = self.toText()
makefile = FileUtil.openFile(self.path)
makefile.writelines(makefileContent)
for subDocument in self.subDocuments:
subDocument.writeToFile()
makefile.close()
def parse_args():
global pluginObj
args=get_args()
script=args.script
exp=args.exp
host=args.host
if script=='list':
pluginObj.showPluginList()
print '[#]usage：'+os.path.basename(__file__)+' -script scriptName'
sys.exit()
if exp=='list':
fileUtil.showPocFileList()
print '[#]usage：'+os.path.basename(__file__)+' -host http://localhost/ -exp cmsexp.poc'
sys.exit()
if host!=None and exp!=None:
executeExp(host,exp)
if script!=None:
use_script(script,host)
def writeToFile(self):
makefileContent = self.toText()
makefile = FileUtil.openFile(self.path)
makefile.writelines(makefileContent)
for subDocument in self.subDocuments:
subDocument.writeToFile()
makefile.close()
def __init__(self):
self.dao = CrawlDao.CrawlDao("modeldatas")
self.fileDao = FileUtil.FileDao()
self.part_code_id_dict = dict()
part_sql = "select id,sum_code from db_category_part where is_deleted = 'N'"
for part_data in self.dao.db.get_data(part_sql):
self.part_code_id_dict[str(part_data['sum_code'])] = int(
part_data['id'])
def __init__(self):
self. fileDao = FileUtil.FileDao()
self.dao = CrawlDao.CrawlDao()
# 获得新老的对应关系
relation_sql_string = 'select my_cat_id,old_cat_id from db_category_relation'
result_array = self.dao.db.get_data(relation_sql_string)
self.relation_table = {}
for relation_result in result_array:
self.relation_table[int(relation_result['old_cat_id'])] = int(relation_result['my_cat_id'])
def getStyle(url):
url_hash = EncryptUtil.md5(url)
# 先检查缓存里面的style
file_path = getFilePath(url_hash)
loadF = None
try:
if not os.path.exists(file_path):
# 不存在，则需要下载
styles = CssUtil.downLoad(url)
if styles:
with open(file_path, u'w') as loadF:
json.dump(
{
u'update_time':
datetime.datetime.now().strftime(
u'%Y-%m-%d %H:%M:%S'),
u'url':
url,
u'styles':
styles
}, loadF)
return styles
else:
with open(file_path, u'r') as loadF:
detail = json.load(loadF)
update_time = detail[u'update_time']
styles = detail[u'styles']
# 如果更新时间之间相差5天，就下载
update_time = datetime.datetime.strptime(
update_time, u'%Y-%m-%d %H:%M:%S')
now = datetime.datetime.now()
space_day = (now - update_time).days
if space_day >= 5:
# 需要重新下载
loadF.close()
FileUtil.delFile(file_path)
return getStyle(url)
else:
# 不需要重新下载
return styles
finally:
if loadF:
loadF.close()
def initKakefile(self):
kakefile = FileUtil.openFile('Kakefile')
kakefileContent = """project:
name: ${name}
version: ${version}
type: cpp"""
kakefile.write(kakefileContent)
kakefile.close()
return True
def checkAllIpsFromFileParallel(fileName):
ips = FileUtil.readIPsFromFile(fileName)
pool = Pool(processes=NO_OF_PROCESS)
result = pool.map_async(getIPPingStatus, ips)
# print(result.get())
pool.close()
dataList = result.get()
# print("======>", type(result.get()))
for i in dataList:
print(i)
def initKakefile(self):
kakefile = FileUtil.openFile('Kakefile')
kakefileContent = """project:
name: ${name}
version: ${version}
type: cpp"""
kakefile.write(kakefileContent)
kakefile.close()
return True
def __init__(self):
self.dao = CrawlDao.CrawlDao("athena_center", "local")
self.fileDao = FileUtil.FileDao()
self.stringUtil = StringUtil.StringUtil()
self.insert_pic_data_list = list()
self.update_goods_car_data_list = list()
# mac pic
self.max_pic_id = 0
# {{pic_num,pic_index}:[oe,oe...]}
self.pic_dict = dict()
def wordEmbedding(self, filename, len_word_vocabulary):
word_embedding = np.zeros((len_word_vocabulary, self.word_vector_len),
dtype=np.float32)
context = FileUtil.readFile(filename)
self.word_vocabulary = {}
for i, c in enumerate(context):
if i >= len_word_vocabulary:
break
wordVector = c.split(" ")
word_embedding[i, :] = np.array([float(t) for t in wordVector[1:]])
self.word_vocabulary[wordVector[0]] = i
return word_embedding
def load_products(self, products_file):
items = FileUtil.load_csv(products_file)
for pid, name, tname, tid, year, pdf, image in items:
self.products.append(
Product(product_id=pid,
name=name,
theme=tid,
store_folder=self.store_folder,
theme_name=tname,
launch_year=year,
pdf_locations=[pdf],
images=[image]))
def create_model(self):
base_dir = 'best/all/'
fileUtil = FileUtil()
for f in listdir(base_dir):
str_file = base_dir+f
print '****************************', str_file
with codecs.open(str_file, 'r', 'utf-8') as file:
for f in file.readlines():
lst = f.split('|')
for i in range(len(lst) - 1):
j = 0
str = lst[i]
# print str
lst_char = str[:]
for j in range(len(lst_char)):
if j == 0:
ans = 'B'
else:
ans = 'I'
message_model = lst_char[j]+' '+crf.get_group(lst_char[j])+' '+ans
print message_model
fileUtil.write_file('train', message_model)
def make(self, projectConfig):
makeConf = {
'target_type': projectConfig.getItem('make.configuration.target.type', self.defaultConf['target_type']),
'ar': projectConfig.getItem('make.configuration.archiver.ar', self.defaultConf['ar']),
'ld': projectConfig.getItem('make.configuration.linker.ld', self.defaultConf['ld']),
'ldflags': projectConfig.getItem('make.configuration.linker.flags', self.defaultConf['ldflags']),
'ld_library_paths': projectConfig.getItem('make.configuration.linker.library_paths', self.defaultConf['ld_library_paths']),
'libraries': projectConfig.getItem('make.configuration.linker.libraries', []),
'solution_dir': projectConfig.getItem('global.solution.dir', '..'),
'sources': projectConfig.getItem('make.configuration.linker.include', []),
}
makefile = GMakeDocument(TARGET_MAKEFILE_PATH)
objectSourcePaths = []
self.objectFiles.clear()
solutionDirPath = Path(makeConf['solution_dir'], True)
objectFilePaths = []
for objectSourcePath in makeConf['sources']:
objectSourcePath = solutionDirPath.join(objectSourcePath)
FileUtil.searchAllFiles(objectFilePaths, objectSourcePath, [ '.o' ])
for objectFilePath in objectFilePaths:
self.objectFiles.append(Path(objectFilePath))
finalFilePath = self.getFinalFilePath(projectConfig, makeConf)
linkRule = self.getLinkRule(finalFilePath, makeConf)
subMakefile = GMakeDocument(FINAL_TARGET_SUBMAKE_PATH)
subMakefile.addRule(linkRule)
makefile.addSubDocument(subMakefile)
allRule = GMakeSimpleRule('all', [finalFilePath])
makefile.addRule(allRule)
makefile.writeToFile()
return True
def crfpp(self, msg, model):
crf = CRF()
fileUtil = FileUtil()
crf.create_file_input(msg)
start_time = time.time()
os.system('crf_test -m '+model+' crf.test.data > logs/out/crf.result')
total_time = time.time() - start_time #return in seconds
lst = fileUtil.read_file('logs/out/crf.result')
# lst = [a for a in lst if a != u'\n']
# str_ans = reduce(lambda x,y:x+y, [a.split('\t')[0] for a in lst])
# ans = reduce(lambda x,y:x+y, [a.split('\t')[3][:-1] for a in lst])
# lst_col3 = [a.split('\t')[3][:-1] for a in lst]
lst_col3, str_ans = self.process_ans(lst)
lst_ans = [n for (n, e) in enumerate(lst_col3) if e == 'B']
result_lst = []
for i in range(len(lst_ans)-1):
a = lst_ans[i]
b = lst_ans[i+1]
result_lst.append(str_ans[a:b])
result_lst.append(str_ans[b:len(str_ans)])
return total_time, result_lst
def initKakefile(self):
kakefile = FileUtil.openFile('Kakefile')
kakefileContent = """project:
name: ${name}
version: ${version}
type: ld
make:
configuration:
linker:
include: [ ]"""
kakefile.write(kakefileContent)
kakefile.close()
return True
def make(self, projectConfig):
makeConf = {
'target_type': projectConfig.getItem('make.configuration.target.type', self.defaultConf['target_type']),
'cc': projectConfig.getItem('make.configuration.compiler.c.cc', self.defaultConf['cc']),
'cxxc': projectConfig.getItem('make.configuration.compiler.cpp.cc', self.defaultConf['cxxc']),
'cflags': projectConfig.getItem('make.configuration.compiler.c.flags', self.defaultConf['cflags']),
'cxxflags': projectConfig.getItem('make.configuration.compiler.cpp.flags', self.defaultConf['cxxflags']),
'fpic': projectConfig.getItem('make.configuration.compiler.fpic', self.defaultConf['fpic']),
'autolink': projectConfig.getItem('make.configuration.linker.autolink', self.defaultConf['autolink']),
'ar': projectConfig.getItem('make.configuration.archiver.ar', self.defaultConf['ar']),
'ld': projectConfig.getItem('make.configuration.linker.ld', self.defaultConf['ld']),
'ldflags': projectConfig.getItem('make.configuration.linker.flags', self.defaultConf['ldflags']),
'ld_library_paths': projectConfig.getItem('make.configuration.linker.library_paths', self.defaultConf['ld_library_paths']),
'libraries': projectConfig.getItem('make.configuration.linker.libraries', []),
'c_src_exts': projectConfig.getItem('make.configuration.compiler.c.src_exts', self.defaultConf['c_src_exts']),
'cxx_src_exts': projectConfig.getItem('make.configuration.compiler.cpp.src_exts', self.defaultConf['cxx_src_exts']),
'c_include_paths': projectConfig.getItem('make.configuration.compiler.c.include_paths', self.defaultConf['c_include_paths']),
'cxx_include_paths': projectConfig.getItem('make.configuration.compiler.cpp.include_paths', self.defaultConf['cxx_include_paths']),
'cxx_using_c_include_paths': projectConfig.getItem('make.configuration.compiler.cpp.inherit_c_include_path', self.defaultConf['cxx_using_c_include_paths'])
}
makefile = GMakeDocument(TARGET_MAKEFILE_PATH)
self.mainSourceFiles.clear()
self.objectFiles.clear()
cSourceFiles = []
cppSourceFiles = []
FileUtil.searchAllFiles(cSourceFiles, SRC_MAIN_PATH, makeConf['c_src_exts'])
FileUtil.searchAllFiles(cppSourceFiles, SRC_MAIN_PATH, makeConf['cxx_src_exts'])
self.mainSourceFiles.extend(cSourceFiles)
self.mainSourceFiles.extend(cppSourceFiles)
for fileName in self.mainSourceFiles:
filePath = Path(fileName, True)
relPath = filePath.getRelevantPath(SRC_MAIN_PATH)
dirName = relPath.getDirName()
baseName = relPath.getBaseName()
subMakeDirPath = TARGET_SUBMAKE_MAIN_PATH.join(dirName)
FileUtil.createDirectory(subMakeDirPath)
subMakefilePath = subMakeDirPath.join(PathUtil.getPrefix(baseName))
subMakefilePath.appendExt('mk')
subMakefile = GMakeDocument(subMakefilePath)
compileRule = GCCCompileRule(filePath, [INCLUDE_MAIN_PATH], SRC_MAIN_PATH, TARGET_OBJECT_MAIN_PATH, makeConf)
subMakefile.addRule(compileRule)
makefile.addSubDocument(subMakefile)
objectFilePath = Path(compileRule.getTarget())
self.objectFiles.append(objectFilePath)
if makeConf['autolink']:
if makeConf['target_type'] == 'executable':
subMakefile = GMakeDocument(FINAL_TARGET_SUBMAKE_PATH)
finalFileName = '%(name)s-%(version)s' % {
'name': projectConfig.getItem('project.name', 'noname'),
'version': projectConfig.getItem('project.version', '1.0.0')}
finalFilePath = TARGET_PATH.join(finalFileName)
linkRule = GCCLinkRule(finalFilePath, self.objectFiles, makeConf)
subMakefile.addRule(linkRule)
makefile.addSubDocument(subMakefile)
allRule = GMakeSimpleRule('all', [finalFilePath])
makefile.addRule(allRule)
elif makeConf['target_type'] == 'dynamic_library':
subMakefile = GMakeDocument(FINAL_TARGET_SUBMAKE_PATH)
finalFileName = 'lib%(name)s.so.%(version)s' % {
'name': projectConfig.getItem('project.name', 'noname'),
'version': projectConfig.getItem('project.version', '1.0.0')}
finalFilePath = TARGET_PATH.join(finalFileName)
linkRule = GCCLinkRule(finalFilePath, self.objectFiles, makeConf)
subMakefile.addRule(linkRule)
makefile.addSubDocument(subMakefile)
allRule = GMakeSimpleRule('all', [finalFilePath])
makefile.addRule(allRule)
elif makeConf['target_type'] == 'static_library':
subMakefile = GMakeDocument(FINAL_TARGET_SUBMAKE_PATH)
finalFileName = 'lib%(name)s.a.%(version)s' % {
'name': projectConfig.getItem('project.name', 'noname'),
'version': projectConfig.getItem('project.version', '1.0.0')}
finalFilePath = TARGET_PATH.join(finalFileName)
arRule = ArRule(finalFilePath, self.objectFiles, makeConf)
subMakefile.addRule(arRule)
makefile.addSubDocument(subMakefile)
allRule = GMakeSimpleRule('all', [finalFilePath])
makefile.addRule(allRule)
else:
self.logger.warn('target_type is not recognized!')
sys.exit(1)
else:
allRule = GMakeSimpleRule('all', self.objectFiles)
makefile.addRule(allRule)
makefile.writeToFile()
return True
def init(self):
FileUtil.createDirectory(SRC_PATH)
FileUtil.createDirectory(INCLUDE_PATH)
FileUtil.createDirectory(TARGET_PATH)
FileUtil.createDirectory(SRC_MAIN_PATH)
FileUtil.createDirectory(SRC_TEST_PATH)
FileUtil.createDirectory(INCLUDE_MAIN_PATH)
FileUtil.createDirectory(INCLUDE_TEST_PATH)
return True
def init(self):
FileUtil.createDirectory(TARGET_PATH)
return True
