def nltk_permutations(plot, models, workers):
from nltk.corpus import brown, reuters, gutenberg, genesis, webtext
cs_n = [('brown', brown), ('reuters', reuters), ('gutenberg', gutenberg),
('genesis', genesis), ('webtext', webtext)]
cs_nc = [(n, c.raw()) for n, c in cs_n]
cs_perm = list(permutations(cs_nc))
ols, avs, avpd, vs = [], [], [], []
for csp in tqdm(cs_perm, desc='NLTK permutations'):
names, corpora = list(zip(*csp))
folder = os.path.join('models', 'nltk_permutations',
'_'.join([n[:2] for n in names]))
createFolder(folder)
with open(os.path.join(folder, 'corpus_list.txt'), 'w') as f:
f.write('\n'.join([n for n in names]))
save_path = os.path.join(folder, 'nltk_w2v.model')
if plot:
models = 'load'
no_metrics_save = True
else:
no_metrics_save = False
if models == 'load':
corpora = save_path
try: # plot from existing metrics.jsons
order_locals, avg_speeds, avg_pw_dists, vocablens = \
emb.main(corpora, save_path=save_path,
lang='english', size=300, window=5, min_count=1, workers=workers,
epochs=20, max_vocab_size=None, n_neighbors=10,
models=models, plot=False, std=False, no_metrics_save=no_metrics_save,
samplesize=10000)
except:
print(corpora, "doesn't exist.")
continue
elif models == 'train':
# Resume training
if os.path.exists(os.path.join(folder, 'metrics.json')):
print(folder, "DONE")
continue
else:
order_locals, avg_speeds, avg_pw_dists, vocablens = \
emb.main(corpora, save_path=save_path,
lang='english', size=300, window=5, min_count=1, workers=workers,
epochs=20, max_vocab_size=None, n_neighbors=10,
models=models, plot=False, std=False, no_metrics_save=no_metrics_save,
samplesize=10000)
ols.append(order_locals)
avs.append(avg_speeds)
avpd.append(avg_pw_dists)
vs.append(vocablens)
if plot:
emb.plot_sos_metrics(order_locals=ols,
avg_speeds=avs,
avg_pw_dists=avpd,
vocablens=vs)
def origo(plot, models, workers, max_vocab_size=200000, min_count=100):
data = 'crawlers/origo.jl'
folder = 'models/origo'
createFolder(folder)
emb.main(data,
save_path=os.path.join(folder, 'origo_w2v.model'),
lang='hungarian',
size=300,
window=5,
workers=workers,
epochs=20,
n_neighbors=10,
models=models,
plot=plot,
std=False,
no_metrics_save=False,
max_vocab_size=max_vocab_size,
min_count=min_count)
def experiment():
baseResultsFolder = '{}/ML/Results/\'COLUMN\''.format(os.getcwd())
util.renameFolder(baseResultsFolder.replace('\'COLUMN\'', 'MINIMAL'),
baseResultsFolder.replace('\'COLUMN\'', 'MINIMAL_old'))
util.renameFolder(
baseResultsFolder.replace('\'COLUMN\'', 'EQUIVALENT'),
baseResultsFolder.replace('\'COLUMN\'', 'EQUIVALENT_old'))
arguments = ['', '--all']
for iCount in range(30):
print('Execução: {}'.format(iCount + 1))
# Create base results folder
util.createFolder(baseResultsFolder.replace('\'COLUMN\'', 'MINIMAL'))
util.createFolder(baseResultsFolder.replace('\'COLUMN\'',
'EQUIVALENT'))
# Experiment
debug_main(arguments)
# Rename results folder to include iteration number
util.renameFolder(
baseResultsFolder.replace('\'COLUMN\'', 'MINIMAL'),
baseResultsFolder.replace('\'COLUMN\'',
'MINIMAL_{}'.format(iCount + 1)))
util.renameFolder(
baseResultsFolder.replace('\'COLUMN\'', 'EQUIVALENT'),
baseResultsFolder.replace('\'COLUMN\'',
'EQUIVALENT_{}'.format(iCount + 1)))
# Rename old folder to base folder
util.renameFolder(baseResultsFolder.replace('\'COLUMN\'', 'MINIMAL_old'),
baseResultsFolder.replace('\'COLUMN\'', 'MINIMAL'))
util.renameFolder(
baseResultsFolder.replace('\'COLUMN\'', 'EQUIVALENT_old'),
baseResultsFolder.replace('\'COLUMN\'', 'EQUIVALENT'))
def executeProteum(baseFolder, sourceFile, sessionName, executableFile,
executionType, directory, units):
if not util.pathExists("{}/log".format(baseFolder)):
util.createFolder("{}/log".format(baseFolder))
#####################
# Create Test Session
#####################
print('\n##### \tCriando sessão de testes ' + util.formatNow() + '\t#####')
# Inclui o arquivo driver.c na compilação caso ele exista na pasta
driver = 'driver.c' if util.pathExists(
'{}/driver.c'.format(baseFolder)) else ''
proteum.createSession(baseFolder, sessionName, executionType, sourceFile,
executableFile, directory, driver)
###################
# Create Test Cases
###################
print('\n##### \tCriando casos de teste ' + util.formatNow() + '\t#####')
testCases = util.convertStringToArray(
util.getContentFromFile("{}/testset.txt".format(baseFolder)), "\n")
proteum.createTestCases(sessionName, directory, testCases)
#################
# Show Test Cases
#################
print('\n##### \tCasos de teste ' + util.formatNow() + '\t#####')
print(proteum.showTestCases(sessionName, directory))
###################
# Delete Test Cases
###################
#print ('\n##### \tCasos de teste deletados ' + formatNow() + '\t#####')
#proteum.deleteTestCase(sessionName, directory)
##################
# Generate mutants
##################
print('\n##### \tGerando mutantes ' + util.formatNow() + '\t#####')
proteum.createEmptySetMutants(sessionName, directory)
for unitName in str(units).splitlines():
proteum.generateMutants(sessionName, directory, unitName)
#################
# Execute mutants
#################
print('\n##### \tExecutando mutantes ' + util.formatNow() + '\t#####')
proteum.executeMutants(sessionName, directory)
###############################
# Seta mutante como equivalente
###############################
print('\n##### \tSetando mutantes como equivalentes ' + util.formatNow() +
'\t#####')
proteum.setEquivalent(sessionName, directory, "")
#########################################
# Execute mutants considering equivalents
#########################################
print('\n##### \tExecutando mutantes considerando os equivalentes ' +
util.formatNow() + '\t#####')
proteum.executeMutants(sessionName, directory)
#########################
# Casos de teste efetivos
#########################
print('\n##### \tExibindo casos de teste efetivos ' + util.formatNow() +
'\t#####')
proteum.listGood(sessionName, directory, "i")
####################
# Gera os relatórios
####################
print('\n##### \tGeração dos relatórios ' + util.formatNow() + '\t#####')
proteum.showMutants(sessionName, directory, "")
###################
# Exibe os mutantes
###################
print('\n##### \tExibe mutantes ' + util.formatNow() + '\t#####')
proteum.generateReport(sessionName, directory)
},
"pvs-8": {
"description": "palio-2",
"folder": 'E:\\DataSets\\100 Hz\\PVS 8\\',
"ini": "2019-12-26T18:51:04",
"end": "2019-12-26T19:11:17"
},
"pvs-9": {
"description": "palio-3",
"folder": 'E:\\DataSets\\100 Hz\\PVS 9\\',
"ini": "2019-12-26T19:23:32",
"end": "2019-12-26T19:38:31"
},
}
######################################### CHANGE HERE #########################################
dataset = workspace['pvs-1']
# side = "left"
side = "right"
###############################################################################################
folder = dataset['folder']
ini_video = parseDateToTimestamp(dataset['ini'])
end_video = parseDateToTimestamp(dataset['end'])
data_file = os.path.join(folder, "dataset_gps_mpu_" + side + ".csv")
video_file = "video_dataset_" + side + ".mp4"
videos_folder = os.path.join(folder, "videos_" + side)
createFolder(videos_folder)