def startWriteCfg(version="debug", force_publish=False):
cfg_excel_name = "cfgExcel_" + version + ".xlsx" #拼接出带版本号的excel文件
local_cfg_path = os.path.join(rootConfig.LOCAL_EXCEL_PATH,
cfg_excel_name) #获得本地的excel文件，删掉
svn_cfg_path = rootConfig.SVN_EXCEL_PATH + cfg_excel_name # 拼接svn上的excel文件地址
utils.createPath(rootConfig.LOCAL_EXCEL_PATH)
#强制使用最新的配置
if force_publish:
utils.svnExportForce(svn_cfg_path, rootConfig.LOCAL_EXCEL_PATH,
local_cfg_path)
else:
utils.svnExport(svn_cfg_path, rootConfig.LOCAL_EXCEL_PATH,
local_cfg_path)
if utils.checkPathExists(local_cfg_path):
global excel_data
excel_data = ParserExcel(local_cfg_path)
writeProjCfg()
writeBaseCfg()
writeUpdateInfo()
writeDomainMap()
print(u"parser config done!")
else:
print(u"get cfgExcel.xlsx failure!!!")
def install(self, filename, directory = ""):
"""Installe le packet dans le repertoire specifie"""
try:
os.mkdir(directory)
except:
self.LOG.warning(f"Destination directory already exists '{directory}'")
with open(filename, "rb") as packet:
if not self.read(packet):
return False
else:
for path, size in self.INDEX_TABLE.table.items():
utils.createPath(os.path.join(directory, path))
# on lit et ecrit les donnees
with open(os.path.join(directory, path), "wb") as doc:
# on lis les donnees du packets
to_read = size // (1024 * 32)
remainder = size % (1024 * 32)
for _ in range(to_read):
datas = packet.read(1024 * 32)
doc.write(datas)
doc.write(packet.read(remainder))
# et flush
doc.flush()
# sans erreurs ;)
self.LOG.info("Packet installed successfully!")
return True
def Main(filesMap, outputdir, limit):
listaSessoes = filesMap[dados.kVotacaoCandidatos()]
fileVotacao = os.path.join(outputdir, "votacoes.txt")
utils.createPath(fileVotacao)
for filename in listaSessoes:
parseVotacao(filename, fileVotacao, limit)
def createSendFile(content):
utils.createPath(globals.path_text_file)
if (not utils.validatePathExistance(globals.path_text_file + "/" + globals.text_file)):
with open(globals.path_text_file + "/" + globals.text_file, 'w') as newfile:
newfile.write(content)
newfile.close()
print("Arquivo com nome '{name}' criado com sucesso na pasta com caminho: '{path}'.\nArquivo criado com sucesso!".format(name=globals.text_file, path=globals.path_text_file))
return True
else:
print("Arquivo com nome '{name}' não foi criado na pasta com caminho: '{path}'.\nArquivo já existente!".format(name=globals.text_file, path=globals.path_text_file))
def Main(filesMap, outputdir, limit):
listaCandidatos = filesMap[dados.kCandidatos()]
fileCandidatos = os.path.join(outputdir, "candidatos.txt")
fileEleicoes = os.path.join(outputdir, "eleicoes.txt")
utils.createPath(fileCandidatos)
for filename in listaCandidatos:
parseCandidatos(filename, fileCandidatos, limit)
parseEleicao(filename, fileEleicoes, limit)
def Main(filesMap, outputdir, limit):
listaPerfilEleitores = filesMap[dados.kPerfilEleitores()]
fileMasterData = os.path.join(outputdir, "masterData.txt")
fileMunicipios = os.path.join(outputdir, "municipios.txt")
fileZonasEleitorais = os.path.join(outputdir, "zonasEleitorais.txt")
fileDemografia = os.path.join(outputdir, "DadosDemograficos.txt")
utils.createPath(fileMunicipios)
for filename in listaPerfilEleitores:
parseMasterData(filename, fileMasterData, limit)
parseMunicipios(filename, fileMunicipios, limit)
parseZonasEleitorais(filename, fileZonasEleitorais, limit)
parseDemografia(filename, fileDemografia, limit)
def glove_embedding(filename, vocab_file, cooccurence_file, domain):
gv = Glove()
out_dir = './preprocessed_data/' + domain
if vocab_file and cooccurence_file:
vocab = gv.load_vocab_in_order(vocab_file)
cooccurence = gv.load_cooccurence_matrix(cooccurence_file)
logger.info('get pre-trained glove embedding')
original_embedding = gv.get_original_embedding(
'./pretrained_embeddings/glove.6B/glove.6B.300d.txt')
mittens_model = Mittens(n=300, max_iter=1000)
logger.info('Start fine tuning...')
new_embeddings = mittens_model.fit(
cooccurence,
vocab=vocab,
initial_embedding_dict=original_embedding)
fin = open(out_dir + '/fine_tuned_glove_300', 'wb')
pickle.dump(new_embeddings, fin)
fin.close()
logger.info('Fine tuning complete')
else:
logger.info('Load english data')
fin = codecs.open(filename, 'r', 'utf-8')
corpus = []
for line in fin:
corpus.append(line)
vocab = gv.build_vocab(corpus)
vocab_file = out_dir + '/vocab.pkl'
createPath(vocab_file)
outfile = open(vocab_file, 'wb')
pickle.dump(vocab, outfile)
outfile.close()
logger.info("Fetching cooccurrence list..")
cooccurrences = gv.build_cooccur(vocab, corpus)
cooccurrences = gv.convert_cooccurence_matrix(cooccurrences,
len(vocab))
cooccurrence_file = out_dir + '/cooccurrence.pkl'
outfile = open(cooccurrence_file, 'wb')
pickle.dump(cooccurrences, outfile)
outfile.close()
logger.info("Cooccurrence list fetch complete (%i pairs).\n",
cooccurrences.shape[0])
def createFilesPath():
utils.createPath("files/" + globals.current_user_nickname + "/send_to")
utils.createPath("files/" + globals.current_user_nickname + "/received_from")