def main(argv=None):
# cfg = base+str(i+1)+'.py'
cfg = 'config.py'
if argv is not None:
sys.argv = argv
opt, args = fn.get_args(sys.argv)
if args is not None: cfg = args[0]
if 's' in opt:
out.level = 0
if 'v' in opt:
out.level = 2
if 'd' in opt:
DEBUG = True
out.level = 3
out(1, '~~~ DEBUG MODE ~~~')
if 'e' in opt:
import prepare
prepare.main(['_3_fitmof.py', '-ce', cfg])
if 'h' in opt:
out(1, 'No help page yet!')
return 0
out(1, 'Begin moffat characterization')
MOF_INIT = None
MAX_IT = 0
SHOW = False
f = open(cfg, 'r')
exec f.read()
f.close()
vars = [
'FILENAME', 'NOWRITE', 'STARS', 'NPIX', 'S_FACT', 'G_RES', 'IMG_GAIN',
'SIGMA_SKY', 'SKY_BACKGROUND'
]
err = fn.check_namespace(vars, locals())
if err > 0:
out(3, 'Error in file', cfg)
return 1
out(2, 'Restore data from extracted files')
files, cat = ws.get_files(FILENAME, 'img') #@UndefinedVariable
fnb = len(cat)
data = ws.restore(*ws.getfilenames(fnb))
data['filenb'] = fnb
data['starnb'] = len(STARS) #@UndefinedVariable
out(1, 'Begin moffat fit on', FILENAME) #@UndefinedVariable
mpar = []
for i in xrange(fnb):
out(1, '===============', i + 1, '/', fnb, '===============')
out(1, 'Working on', files[i])
mofpar, r = fitmof(i, data, locals())
mpar += [mofpar.tolist()]
out(1, '------------------------------------------')
out(1, 'Moffat fit done')
if NOWRITE is False: #@UndefinedVariable
fn.write_cfg(cfg, {'MOF_PARAMS': mpar})
return 0
def main(argv=None):
cfg = 'config.py'
if argv is not None:
sys.argv = argv
opt, args = fn.get_args(sys.argv)
if args is not None: cfg = args[0]
if 's' in opt:
out.level = 0
if 'v' in opt:
out.level = 2
if 'd' in opt:
DEBUG = True
out.level = 3
out(1, '~~~ DEBUG MODE ~~~')
if 'e' in opt:
import prepare
prepare.main(['_3_fitmof.py', '-ce', cfg])
if 'h' in opt:
out(1, 'No help page yet!')
return 0
out(1, 'Begin gaussian fit')
SHOW = None
f = open(cfg, 'r')
exec f.read()
f.close()
vars = ['FILENAME', 'STARS','NPIX', 'MOF_PARAMS',
'G_PARAMS', 'G_POS', 'G_STRAT', 'S_FACT', 'NOWRITE',
'G_RES', 'CENTER', 'IMG_GAIN', 'SIGMA_SKY',
'SKY_BACKGROUND', 'G_SETTINGS', 'NB_RUNS', 'FIT_RAD']
err = fn.check_namespace(vars, locals())
if err > 0:
return 1
out(2, 'Restore data from extracted files')
files, cat = ws.get_files(FILENAME, 'img') #@UndefinedVariable
fnb = len(cat)
data = ws.restore(*ws.getfilenames(fnb))
data['filenb'] = fnb
data['starnb'] = len(STARS) #@UndefinedVariable
gpar = []
gpos = []
for i in xrange(fnb):
out(1, '===============', i+1, '/', fnb,'===============')
out(1, 'Working on', files[i])
gauspar, gauspos = fitgaus(i, data, locals())
gpar += [gauspar]
gpos += [gauspos]
out(1, '------------------------------------------')
out(1, 'Gaussian fit done')
if NOWRITE is False: #@UndefinedVariable
fn.write_cfg(cfg, {'G_PARAMS':gpar, 'G_POS':gpos})
return 0
def main(argv=None):
cfg = 'config.py'
if argv is not None:
sys.argv = argv
opt, args = fn.get_args(sys.argv)
MAX_IT_D = MAXPOS_STEP = MAX_IRATIO_STEP = SHOW = FORCE_INI = None
if args is not None: cfg = args[0]
if 's' in opt:
out.level = 0
if 'v' in opt:
out.level = 2
if 'd' in opt:
DEBUG = True
out.level = 3
out(1, '~~~ DEBUG MODE ~~~')
if 'b' in opt:
import prepare
prepare.main(['deconv.py', '-b', cfg])
if 'e' in opt:
import prepare
prepare.main(['deconv.py', '-ce', cfg])
if 'i' in opt:
FORCE_INI = True
if 'h' in opt:
out(1, 'No help page yet!')
return 0
out(1, 'Begin deconvolution process')
#TODO: check workspace
f = open(cfg, 'r')
exec f.read()
f.close()
vars = [
'FILENAME', 'MAX_IT_D', 'S_FACT', 'G_RES', 'SIGMA_SKY', 'MOF_PARAMS',
'G_STRAT', 'G_PARAMS', 'G_POS', 'CENTER'
]
err = fn.check_namespace(vars, locals())
if err > 0:
return 1
out(2, FILENAME) #@UndefinedVariable
out(2, 'Restore data from extracted files')
fnb = ws.get_filenb(FILENAME) #@UndefinedVariable
files = ws.getfilenames(fnb)
data = ws.restore(*files)
data['filenb'] = fnb
dec = deconv(data, locals())
# if NOWRITE is False:
# fn.write_cfg(cfg, {'DEC_PAR':dec})
out(1, 'Deconvolution done')
return 0
#!/usr/bin/env python
# coding:utf-8
import prepare
import character
import plot
if __name__ == "__main__":
Stage = "29010"
D = plot.DefaultPickList
prepare.main()
for Pick in [
D["MeleePhysical"], D["MeleeMagic"], D["RangedPhysical"],
D["RangedMagic"], D["Control"]
]:
for Enemy in character.enemy_dict.keys():
PickList = Pick["PickList"]
PickListName = Pick["PickListName"]
Baseline = Pick["Baseline"]
ShowSlayLine = Pick.get("ShowSlayLine", True)
MultiTarget = Pick.get("MultiTarget", True)
IgnorePolish = Pick.get("IgnorePolish", False)
plot.plot(Stage,
PickList,
PickListName,
Baseline,
Enemy,
show_slay_line=ShowSlayLine,
multi_target=MultiTarget,
ignore_polish=IgnorePolish)
def main(defaults):
"""
Updates PRONOM signatures
Interactive script, requires keyboard input
"""
try:
resume_download = False
answers = ['y','yes']
versionXML = defaults['versionXML'].format("{0}","{1}",defaults['containerVersion'],defaults['fidoSignatureVersion'],defaults['version'])
#print versionXML
print "FIDO signature updater v"+defaults['version']
print "Contacting PRONOM..."
currentVersion = getPronomSignature("version")
if currentVersion == False:
print "Failed to obtain PRONOM signature file version number, please try again"
sys.exit()
print "Querying latest signaturefile version..."
signatureFile = os.path.join(os.path.abspath(defaults['conf_dir']), defaults['signatureFileName'].format(currentVersion))
if os.path.isfile(signatureFile):
print "You already have the latest PRONOM signature file, version "+str(currentVersion)
ask = raw_input("Update anyway? (yes/no): ")
if ask.lower() not in answers:
sys.exit()
print "Downloading signature file version "+str(currentVersion)+"..."
currentFile = getPronomSignature("file")
if currentFile == False:
print "Failed to obtain PRONOM signature file, please try again"
exit()
sigfile = open(signatureFile,'wb')
sigfile.write(currentFile)
sigfile.close()
print "Writing {0}...".format(defaults['signatureFileName'].format(currentVersion))
print "Extracting PRONOM PUID's from signature file..."
tree = CET.parse(signatureFile)
puids = []
for node in tree.iter("{http://www.nationalarchives.gov.uk/pronom/SignatureFile}FileFormat"):
puids.append(node.get("PUID"))
numberPuids = len(puids)
print "Found "+str(numberPuids)+" PRONOM PUID's"
print "Downloading signatures can take a while"
ask = raw_input("Continue and download signatures? (yes/no): ")
if ask.lower() not in answers:
print "Aborting update..."
sys.exit()
tmpdir = os.path.join(os.path.abspath(defaults['conf_dir']), defaults['tmp_dir'])
if os.path.isdir(tmpdir):
print "Found previously created temporary folder for download:", tmpdir
ask = raw_input("Resume download (yes) or start over (no)?: ")
if ask.lower() in answers:
print "Resuming download..."
resume_download = True
else:
resume_download = False
else:
print "Creating temporary folder for download:", tmpdir
try:
os.mkdir(tmpdir)
except:
pass
if not os.path.isdir(tmpdir):
tmpdir = os.path.join(os.path.abspath(defaults['conf_dir']))
print "Failed to create temporary folder for PUID's, using", tmpdir
print "Downloading signatures, one moment please..."
one_percent = (float(numberPuids) / 100)
numfiles = 0
for puid in puids:
puidType, puidNum = puid.split("/")
puidFileName = "puid."+puidType+"."+puidNum+".xml"
filename = os.path.join(tmpdir, puidFileName)
if os.path.isfile(filename) and checkWellFormedness(filename) and resume_download is not False:
numfiles += 1
continue
puidUrl = "http://www.nationalarchives.gov.uk/pronom/"+puid+".xml"
try:
filehandle = urllib.urlopen(puidUrl)
except Exception, e:
print "Failed to download signaturefile:", puidUrl
print "Error:", str(e)
print "Please restart and resume download"
sys.exit()
puidfile = open(filename,'wb')
for lines in filehandle.readlines():
puidfile.write(lines)
puidfile.close()
filehandle.close()
if not checkWellFormedness(filename):
os.unlink(filename)
continue
numfiles += 1
percent = int(float(numfiles) / one_percent)
print "\r",
print str(percent)+"%",
time.sleep(defaults['http_throttle'])
print "100%"
try:
import zlib
compression = zipfile.ZIP_DEFLATED
except:
compression = zipfile.ZIP_STORED
modes = {zipfile.ZIP_DEFLATED: 'deflated', zipfile.ZIP_STORED: 'stored'}
print "Creating PRONOM zip,",
zf = zipfile.ZipFile(os.path.join(os.path.abspath(defaults['conf_dir']), defaults['pronomZipFileName'].format(currentVersion)), mode='w')
print "adding files with compression mode '"+modes[compression]+"'"
for puid in puids:
puidType, puidNum = puid.split("/")
puidFileName = "puid."+puidType+"."+puidNum+".xml"
filename = os.path.join(os.path.abspath(defaults['conf_dir']), defaults['tmp_dir'], puidFileName)
if os.path.isfile(filename):
zf.write(filename, arcname=puidFileName, compress_type=compression)
os.unlink(filename)
zf.close()
print "Deleting temporary folder and files..."
try:
for root, dirs, files in os.walk(tmpdir, topdown=False):
for name in files:
os.remove(os.path.join(root, name))
for name in dirs:
os.rmdir(os.path.join(root, name))
os.rmdir(tmpdir)
except:
pass
# update versions.xml
versionsFile = os.path.join(os.path.abspath(defaults['conf_dir']), defaults['versionsFileName'])
print "Updating {0}...".format(defaults['versionsFileName'])
xmlversionsfile = open(versionsFile,'wb')
xmlversionsfile.write(versionXML.format(str(currentVersion),"formats-v"+str(currentVersion)+".xml"))
xmlversionsfile.close()
print "Preparing to convert PRONOM formats to FIDO signatures..."
# there should be a check here to handle prepare.main exit() signal (-1/0/1/...)
prepare.main()
print "FIDO signatures successfully updated"
sys.exit()
import tensorflow as tf
import numpy as np
import prepare, sys
X_train, X_lengths, Y_train, dimen, n_steps, X_test, X_test_lengths, y_test, sentences = prepare.main(
)
def _last_relevant(output, length):
batch_size = tf.shape(output)[0]
max_length = int(output.get_shape()[1])
output_size = int(output.get_shape()[2])
index = tf.range(0, batch_size) * max_length + (tf.cast(length, tf.int32) -
1)
flat = tf.reshape(output, [-1, output_size])
relevant = tf.gather(flat, index)
return relevant
def attention(Y, dimen):
# [batch_size x seq_len x dim] -- hidden states
#Y = tf.constant(np.random.randn(batch_size, seq_len, dim), tf.float32)
# [batch_size x dim] -- h_N
#h = tf.constant(np.random.randn(batch_size, dim), tf.float32)
initializer = tf.random_uniform_initializer()
W = tf.get_variable("weights_Y", [dimen, dimen], initializer=initializer)
w = tf.get_variable("weights_w", [dimen], initializer=initializer)
# [batch_size x seq_len x dim] -- tanh(W^{Y}Y)
M = tf.tanh(tf.einsum("aij,jk->aik", Y, W))
# [batch_size x seq_len] -- softmax(Y w^T)
def main():
"""
Updates PRONOM signatures
"""
global defaults
print "FIDO signature updater v"+defaults['version']
print "Contacting PRONOM..."
currentVersion = getPronomSignature("version")
if currentVersion == False:
exit()
print "Querying latest signaturefile version..."
signatureFile = os.path.join(os.path.abspath(defaults['conf_dir']), defaults['signatureFileName'].format(currentVersion))
if os.path.isfile(signatureFile):
print "You already have the latest PRONOM signature file, version "+str(currentVersion)
answers = {'y','yes'}
ask = raw_input("Update anyway? (yes/no): ")
if ask.lower() not in answers:
sys.exit()
print "Downloading signature file version "+str(currentVersion)+"..."
currentFile = getPronomSignature("file")
if currentFile == False:
exit()
sigfile = open(signatureFile,'wb')
sigfile.write(currentFile)
sigfile.close()
print "Extracting PRONOM PUID's from signature file..."
tree = CET.parse(signatureFile)
puids = []
for node in tree.iter("{http://www.nationalarchives.gov.uk/pronom/SignatureFile}FileFormat"):
puids.append(node.get("PUID"))
numberPuids = len(puids)
print "Found "+str(numberPuids)+" PRONOM PUID's"
print "Downloading signatures can take a while"
answers = {'y','yes'}
ask = raw_input("Continue and download signatures? (yes/no): ")
if ask.lower() not in answers:
print "Aborting update..."
sys.exit()
tmpdir = os.path.join(os.path.abspath(defaults['conf_dir']), defaults['tmp_dir'])
print "Creating temporary folder for download:", tmpdir
try:
os.mkdir(tmpdir)
except:
pass
if not os.path.isdir(tmpdir):
tmpdir = os.path.join(os.path.abspath(defaults['conf_dir']))
print "Failed to create temporary folder for PUID's, using", tmpdir
print "Downloading signatures, one moment please..."
one_percent = (float(numberPuids) / 100)
numfiles = 0
for puid in puids:
puidType, puidNum = puid.split("/")
filehandle = urllib.urlopen('http://www.nationalarchives.gov.uk/pronom/'+puid+'.xml')
puidFileName = "puid."+puidType+"."+puidNum+".xml"
filename = os.path.join(os.path.abspath(defaults['conf_dir']), defaults['tmp_dir'], puidFileName)
puidfile = open(filename,'wb')
for lines in filehandle.readlines():
puidfile.write(lines)
puidfile.close()
filehandle.close()
if not checkWellFormedness(filename):
os.unlink(filename)
continue
numfiles += 1
percent = int(float(numfiles) / one_percent)
print "\b\b\b\b\b",
print str(percent)+"%",
time.sleep(defaults['http_throttle'])
print "\b\b\b\b\b",
print "100%"
try:
import zlib
compression = zipfile.ZIP_DEFLATED
except:
compression = zipfile.ZIP_STORED
modes = {zipfile.ZIP_DEFLATED: 'deflated', zipfile.ZIP_STORED: 'stored'}
print "Creating PRONOM zip,",
zf = zipfile.ZipFile(os.path.join(os.path.abspath(defaults['conf_dir']), defaults['pronomZipFile']), mode='w')
print "adding files with compression mode '"+modes[compression]+"'"
for puid in puids:
puidType, puidNum = puid.split("/")
puidFileName = "puid."+puidType+"."+puidNum+".xml"
filename = os.path.join(os.path.abspath(defaults['conf_dir']), defaults['tmp_dir'], puidFileName)
if os.path.isfile(filename):
zf.write(filename, arcname=puidFileName, compress_type=compression)
os.unlink(filename)
zf.close()
print "Deleting temporary folder and files..."
try:
for root, dirs, files in os.walk(tmpdir, topdown=False):
for name in files:
os.remove(os.path.join(root, name))
for name in dirs:
os.rmdir(os.path.join(root, name))
os.rmdir(tmpdir)
except:
pass
print "Preparing to convert PRONOM formats to FIDO signatures..."
prepare.main()
print "FIDO signatures successfully updated"
sys.exit()
def main(defaults):
"""
Updates PRONOM signatures
Interactive script, requires keyboard input
"""
try:
resume_download = False
answers = ['y','yes']
versionXML = defaults['versionXML'].format("{0}","{1}",defaults['containerVersion'],defaults['fidoSignatureVersion'],defaults['version'])
#print versionXML
print "FIDO signature updater v"+defaults['version']
print "Contacting PRONOM..."
currentVersion = getPronomSignature("version")
if currentVersion == False:
print "Failed to obtain PRONOM signature file version number, please try again"
sys.exit()
print "Querying latest signaturefile version..."
signatureFile = os.path.join(os.path.abspath(defaults['conf_dir']), defaults['signatureFileName'].format(currentVersion))
versionsFile = os.path.join(os.path.abspath(defaults['conf_dir']), defaults['versionsFileName'])
if os.path.isfile(signatureFile):
print "You already have the latest PRONOM signature file, version "+str(currentVersion)
ask = raw_input("Update anyway? (yes/no): ")
if ask.lower() not in answers:
sys.exit()
print "Updating {0}...".format(defaults['versionsFileName'])
xmlversionsfile = open(versionsFile,'wb')
xmlversionsfile.write(versionXML.format(str(currentVersion),"formats-v"+str(currentVersion)+".xml"))
xmlversionsfile.close()
print "Downloading signature file version "+str(currentVersion)+"..."
currentFile = getPronomSignature("file")
if currentFile == False:
print "Failed to obtain PRONOM signature file, please try again"
exit()
sigfile = open(signatureFile,'wb')
sigfile.write(currentFile)
sigfile.close()
print "Writing {0}...".format(defaults['signatureFileName'].format(currentVersion))
print "Extracting PRONOM PUID's from signature file..."
tree = CET.parse(signatureFile)
puids = []
for node in tree.iter("{http://www.nationalarchives.gov.uk/pronom/SignatureFile}FileFormat"):
puids.append(node.get("PUID"))
numberPuids = len(puids)
print "Found "+str(numberPuids)+" PRONOM PUID's"
print "Downloading signatures can take a while"
ask = raw_input("Continue and download signatures? (yes/no): ")
if ask.lower() not in answers:
print "Aborting update..."
sys.exit()
tmpdir = os.path.join(os.path.abspath(defaults['conf_dir']), defaults['tmp_dir'])
if os.path.isdir(tmpdir):
print "Found previously created temporary folder for download:", tmpdir
ask = raw_input("Resume download (yes) or start over (no)?: ")
if ask.lower() in answers:
print "Resuming download..."
resume_download = True
else:
resume_download = False
else:
print "Creating temporary folder for download:", tmpdir
try:
os.mkdir(tmpdir)
except:
pass
if not os.path.isdir(tmpdir):
tmpdir = os.path.join(os.path.abspath(defaults['conf_dir']))
print "Failed to create temporary folder for PUID's, using", tmpdir
print "Downloading signatures, one moment please..."
one_percent = (float(numberPuids) / 100)
numfiles = 0
for puid in puids:
puidType, puidNum = puid.split("/")
puidFileName = "puid."+puidType+"."+puidNum+".xml"
filename = os.path.join(tmpdir, puidFileName)
if os.path.isfile(filename) and checkWellFormedness(filename) and resume_download is not False:
numfiles += 1
continue
puidUrl = "http://www.nationalarchives.gov.uk/pronom/"+puid+".xml"
try:
filehandle = urllib.urlopen(puidUrl)
except Exception, e:
print "Failed to download signaturefile:", puidUrl
print "Error:", str(e)
print "Please restart and resume download"
sys.exit()
puidfile = open(filename,'wb')
for lines in filehandle.readlines():
puidfile.write(lines)
puidfile.close()
filehandle.close()
if not checkWellFormedness(filename):
os.unlink(filename)
continue
numfiles += 1
percent = int(float(numfiles) / one_percent)
print "\b\b\b\b\b",
print str(percent)+"%",
time.sleep(defaults['http_throttle'])
print "\b\b\b\b\b",
print "100%"
try:
import zlib
compression = zipfile.ZIP_DEFLATED
except:
compression = zipfile.ZIP_STORED
modes = {zipfile.ZIP_DEFLATED: 'deflated', zipfile.ZIP_STORED: 'stored'}
print "Creating PRONOM zip,",
zf = zipfile.ZipFile(os.path.join(os.path.abspath(defaults['conf_dir']), defaults['pronomZipFileName'].format(currentVersion)), mode='w')
print "adding files with compression mode '"+modes[compression]+"'"
for puid in puids:
puidType, puidNum = puid.split("/")
puidFileName = "puid."+puidType+"."+puidNum+".xml"
filename = os.path.join(os.path.abspath(defaults['conf_dir']), defaults['tmp_dir'], puidFileName)
if os.path.isfile(filename):
zf.write(filename, arcname=puidFileName, compress_type=compression)
os.unlink(filename)
zf.close()
print "Deleting temporary folder and files..."
try:
for root, dirs, files in os.walk(tmpdir, topdown=False):
for name in files:
os.remove(os.path.join(root, name))
for name in dirs:
os.rmdir(os.path.join(root, name))
os.rmdir(tmpdir)
except:
pass
print "Preparing to convert PRONOM formats to FIDO signatures..."
prepare.main()
print "FIDO signatures successfully updated"
sys.exit()
# Prepare # ================================= """ This module performs several steps in preparing the data for analysis. First, it splits each text into segments of a given length. Second, it either takes all segments or samples an equal number of segments per text. Third, it selects the desired features from each segment (form and pos) Fourth, it creates document-term matrixes with absolute, relative and binary feature counts. This function needs to be run again when a parameter is changed. """ segmentlength = 3000 max_num_segments = -1 featuretype = ["lemmata", "NOM"] # forms, pos absolutefreqs, relativefreqs, binaryfreqs = prepare.main(taggedfolder, segmentfolder, datafolder, dtmfolder, segmentlength, max_num_segments, stoplistfile, featuretype) # ================================= # Calculate # ================================= """ This module performs the actual distinctiveness measure for each feature. The calculation can be based on relative or binary features. The calculation can work in several ways: by division, subtraction as well as with or without applying some log transformation. The contrast parameter takes ["category", "group1", "group2"] as in the metadata table. """ separator = ";" contrast = ["group", "Racine", "contemporaries"] # example for roman20
def all_in_one(config):
prepare.main(config)
calc_data.main(config)
calc_data.calc_overall_configured(config)
plot_dets.main(config)
def prepare_experiment(experiment_config):
for cfg in experiment_config:
logging.info("Preparing models to '%s'", cfg.MODELS_DIR)
prepare.main(cfg)
