def main():
params = check_params(get_params())
# Importing Keras et al. takes a few seconds; only do it if params parsed correctly
import transcoder
transcoder.main(**params)
def counterfactual():
arguments = docopt.docopt(__doc__)
dataset = arguments['--dataset']
thought_vector_size = int(arguments['--thought-vector'])
encoder_model = arguments['--encoder']
decoder_model = arguments['--decoder']
classifier_model = arguments['--classifier']
epochs = int(arguments['--epochs'])
decay = float(arguments['--decay'])
learning_rate = float(arguments['--learning-rate'])
gan_type = arguments['--gan-type']
perceptual_layers = int(arguments['--perceptual-layers'])
img_width = int(arguments['--img-width'])
try:
experiment_timestamp = int(arguments['--timestamp'])
except ValueError:
experiment_timestamp = int(time.time())
experiment_name = '-'.join(
str(x) for x in [
dataset,
thought_vector_size,
encoder_model,
decoder_model,
classifier_model,
experiment_timestamp,
])
timestamp = int(time.time())
#Download the dataset if it doesn't already exist
# TODO: security lol
os.system('scripts/download_{}.py'.format(dataset))
train_dataset = os.path.expanduser(
'~/data/{}_train.dataset'.format(dataset))
test_dataset = os.path.expanduser('~/data/{}_test.dataset'.format(dataset))
# Fill params with defaults
import main
defaults = {
opt.long: opt.value
for opt in docopt.parse_defaults(main.__doc__)
}
params = {main.argname(k): main.argval(defaults[k]) for k in defaults}
# TODO: Merge main.py params with these params in some nice elegant way
# Like, imagine if each experiment could inherit all the main.py params
# But it would also have its own params which override the inherited ones
# This is beginning to sound dangerously object-oriented
params['experiment_name'] = experiment_name
params['encoder_input_filename'] = test_dataset
params['decoder_input_filename'] = test_dataset
params['classifier_input_filename'] = test_dataset
params['vocabulary_filename'] = train_dataset
params['encoder_datatype'] = 'img'
params['decoder_datatype'] = 'img'
params['classifier_datatype'] = 'lab'
params['encoder_model'] = encoder_model
params['decoder_model'] = decoder_model
params['discriminator_model'] = encoder_model
params['classifier_model'] = classifier_model
params['thought_vector_size'] = thought_vector_size
params['img_width_encoder'] = img_width
params['img_width_decoder'] = img_width
params['epochs'] = epochs
params['learning_rate'] = learning_rate
params['decay'] = decay
params['perceptual_loss_layers'] = perceptual_layers
params['batches_per_epoch'] = 50
params['enable_classifier'] = True
params['enable_discriminator'] = True
params['enable_perceptual_loss'] = True
params['encoder_weights'] = 'encoder_{}.h5'.format(encoder_model)
params['decoder_weights'] = 'decoder_{}.h5'.format(decoder_model)
params['classifier_weights'] = 'classifier_{}.h5'.format(classifier_model)
params['discriminator_weights'] = 'discriminator_{}.h5'.format(
encoder_model)
params['gan_type'] = gan_type
params['learning_rate_disc'] = learning_rate * .1
params['learning_rate_generator'] = learning_rate * .1
# TODO: security lol
os.system('mkdir ~/results/{}'.format(experiment_name))
save_params(experiment_name, params)
import transcoder
# First train a manifold
train_params = params.copy()
train_params['stdout_filename'] = 'stdout_train_{}.txt'.format(timestamp)
train_params['encoder_input_filename'] = train_dataset
train_params['decoder_input_filename'] = train_dataset
train_params['classifier_input_filename'] = train_dataset
train_params['mode'] = 'train'
if train_params['epochs'] > 0:
transcoder.main(**train_params)
# Evaluate the classifier
"""
eval_params = params.copy()
eval_params['decoder_model'] = params['classifier_model']
eval_params['decoder_datatype'] = params['classifier_datatype']
eval_params['decoder_weights'] = params['classifier_weights']
eval_params['enable_classifier'] = False
eval_params['enable_discriminator'] = False
eval_params['stdout_filename'] = 'stdout_eval_{}.txt'.format(timestamp)
eval_params['mode'] = 'evaluate'
transcoder.main(**eval_params)
"""
# Generate a "dream" video
dream_params = params.copy()
dream_params['video_filename'] = 'dream_output_{}.mjpeg'.format(timestamp)
dream_params['stdout_filename'] = 'stdout_dream_{}.txt'.format(timestamp)
dream_params['enable_discriminator'] = False
dream_params['mode'] = 'dream'
transcoder.main(**dream_params)
# Re-encode the video to mp4 for storage
encode_video(experiment_name, dream_params['video_filename'])
# Add counterfactuals
counter_params = params.copy()
counter_params['video_filename'] = 'counterfactual_output_{}.mjpeg'.format(
timestamp)
counter_params['stdout_filename'] = 'stdout_counterfactual_{}.txt'.format(
timestamp)
counter_params['enable_discriminator'] = False
counter_params['mode'] = 'counterfactual'
for _ in range(3):
transcoder.main(**counter_params)
# Re-encode the video to mp4 for storage
encode_video(experiment_name, counter_params['video_filename'])
# Touch a file to mark the experiment as finished
filename = os.path.expanduser(
'~/results/{}/finished'.format(experiment_name))
open(filename, 'w').write('OK')
sql = """SELECT CR.pk, CR.command, CR.GroupMember
FROM CommandRelationships AS CR
JOIN FileIDs ON CR.fileID=FileIDs.pk
JOIN CommandClassifications ON CR.commandClassification = CommandClassifications.pk
WHERE FileIDs.description='A .pst file'
AND CommandClassifications.classification = 'extract';"""
c, sqlLock = databaseInterface.querySQL(sql)
row = c.fetchone()
while row != None:
ret.append(row)
row = c.fetchone()
sqlLock.release()
#check if not to remove
for extension in removeOnceExtractedSkip:
if fileName.lower().endswith(extension.lower()):
removeOnceExtracted = False
break
return ret
if __name__ == '__main__':
#while 1:
# import time
# time.sleep(10)
transcoder.onSuccess = onceExtracted
transcoder.identifyCommands = identifyCommands
transcoder.replacementDic = replacementDic
filename = sys.argv[1].__str__()
print filename
main(filename)
prefix = opts.fileUUID + "-"
outputDirectory = opts.accessDirectory
elif opts.commandClassifications == "thumbnail":
outputDirectory = opts.thumbnailDirectory
postfix = opts.fileUUID
else:
print >> sys.stderr, "Unsupported command classification."
exit(2)
fileExtensionWithDot = "." + transcoder.fileExtension
if transcoder.fileExtension == "":
fileExtensionWithDot = ""
replacementDic = { \
"%inputFile%": transcoder.fileFullName, \
"%outputDirectory%": outputDirectory, \
"%fileExtension%": transcoder.fileExtension, \
"%fileExtensionWithDot%": fileExtensionWithDot, \
"%fileFullName%": transcoder.fileFullName, \
"%preservationFileDirectory%": transcoder.fileDirectory, \
"%fileDirectory%": transcoder.fileDirectory,\
"%fileTitle%": transcoder.fileTitle, \
"%fileName%": transcoder.fileTitle, \
"%prefix%": prefix,
"%postfix%": postfix
}
transcoder.onSuccess = onceNormalized
transcoder.identifyCommands = identifyCommands
transcoder.replacementDic = replacementDic
main(filename)