def main(argv=None): # cfg = base+str(i+1)+'.py' cfg = 'config.py' if argv is not None: sys.argv = argv opt, args = fn.get_args(sys.argv) if args is not None: cfg = args[0] if 's' in opt: out.level = 0 if 'v' in opt: out.level = 2 if 'd' in opt: DEBUG = True out.level = 3 out(1, '~~~ DEBUG MODE ~~~') if 'e' in opt: import prepare prepare.main(['_3_fitmof.py', '-ce', cfg]) if 'h' in opt: out(1, 'No help page yet!') return 0 out(1, 'Begin moffat characterization') MOF_INIT = None MAX_IT = 0 SHOW = False f = open(cfg, 'r') exec f.read() f.close() vars = [ 'FILENAME', 'NOWRITE', 'STARS', 'NPIX', 'S_FACT', 'G_RES', 'IMG_GAIN', 'SIGMA_SKY', 'SKY_BACKGROUND' ] err = fn.check_namespace(vars, locals()) if err > 0: out(3, 'Error in file', cfg) return 1 out(2, 'Restore data from extracted files') files, cat = ws.get_files(FILENAME, 'img') #@UndefinedVariable fnb = len(cat) data = ws.restore(*ws.getfilenames(fnb)) data['filenb'] = fnb data['starnb'] = len(STARS) #@UndefinedVariable out(1, 'Begin moffat fit on', FILENAME) #@UndefinedVariable mpar = [] for i in xrange(fnb): out(1, '===============', i + 1, '/', fnb, '===============') out(1, 'Working on', files[i]) mofpar, r = fitmof(i, data, locals()) mpar += [mofpar.tolist()] out(1, '------------------------------------------') out(1, 'Moffat fit done') if NOWRITE is False: #@UndefinedVariable fn.write_cfg(cfg, {'MOF_PARAMS': mpar}) return 0
def main(argv=None): cfg = 'config.py' if argv is not None: sys.argv = argv opt, args = fn.get_args(sys.argv) if args is not None: cfg = args[0] if 's' in opt: out.level = 0 if 'v' in opt: out.level = 2 if 'd' in opt: DEBUG = True out.level = 3 out(1, '~~~ DEBUG MODE ~~~') if 'e' in opt: import prepare prepare.main(['_3_fitmof.py', '-ce', cfg]) if 'h' in opt: out(1, 'No help page yet!') return 0 out(1, 'Begin gaussian fit') SHOW = None f = open(cfg, 'r') exec f.read() f.close() vars = ['FILENAME', 'STARS','NPIX', 'MOF_PARAMS', 'G_PARAMS', 'G_POS', 'G_STRAT', 'S_FACT', 'NOWRITE', 'G_RES', 'CENTER', 'IMG_GAIN', 'SIGMA_SKY', 'SKY_BACKGROUND', 'G_SETTINGS', 'NB_RUNS', 'FIT_RAD'] err = fn.check_namespace(vars, locals()) if err > 0: return 1 out(2, 'Restore data from extracted files') files, cat = ws.get_files(FILENAME, 'img') #@UndefinedVariable fnb = len(cat) data = ws.restore(*ws.getfilenames(fnb)) data['filenb'] = fnb data['starnb'] = len(STARS) #@UndefinedVariable gpar = [] gpos = [] for i in xrange(fnb): out(1, '===============', i+1, '/', fnb,'===============') out(1, 'Working on', files[i]) gauspar, gauspos = fitgaus(i, data, locals()) gpar += [gauspar] gpos += [gauspos] out(1, '------------------------------------------') out(1, 'Gaussian fit done') if NOWRITE is False: #@UndefinedVariable fn.write_cfg(cfg, {'G_PARAMS':gpar, 'G_POS':gpos}) return 0
def main(argv=None): cfg = 'config.py' if argv is not None: sys.argv = argv opt, args = fn.get_args(sys.argv) MAX_IT_D = MAXPOS_STEP = MAX_IRATIO_STEP = SHOW = FORCE_INI = None if args is not None: cfg = args[0] if 's' in opt: out.level = 0 if 'v' in opt: out.level = 2 if 'd' in opt: DEBUG = True out.level = 3 out(1, '~~~ DEBUG MODE ~~~') if 'b' in opt: import prepare prepare.main(['deconv.py', '-b', cfg]) if 'e' in opt: import prepare prepare.main(['deconv.py', '-ce', cfg]) if 'i' in opt: FORCE_INI = True if 'h' in opt: out(1, 'No help page yet!') return 0 out(1, 'Begin deconvolution process') #TODO: check workspace f = open(cfg, 'r') exec f.read() f.close() vars = [ 'FILENAME', 'MAX_IT_D', 'S_FACT', 'G_RES', 'SIGMA_SKY', 'MOF_PARAMS', 'G_STRAT', 'G_PARAMS', 'G_POS', 'CENTER' ] err = fn.check_namespace(vars, locals()) if err > 0: return 1 out(2, FILENAME) #@UndefinedVariable out(2, 'Restore data from extracted files') fnb = ws.get_filenb(FILENAME) #@UndefinedVariable files = ws.getfilenames(fnb) data = ws.restore(*files) data['filenb'] = fnb dec = deconv(data, locals()) # if NOWRITE is False: # fn.write_cfg(cfg, {'DEC_PAR':dec}) out(1, 'Deconvolution done') return 0
#!/usr/bin/env python # coding:utf-8 import prepare import character import plot if __name__ == "__main__": Stage = "29010" D = plot.DefaultPickList prepare.main() for Pick in [ D["MeleePhysical"], D["MeleeMagic"], D["RangedPhysical"], D["RangedMagic"], D["Control"] ]: for Enemy in character.enemy_dict.keys(): PickList = Pick["PickList"] PickListName = Pick["PickListName"] Baseline = Pick["Baseline"] ShowSlayLine = Pick.get("ShowSlayLine", True) MultiTarget = Pick.get("MultiTarget", True) IgnorePolish = Pick.get("IgnorePolish", False) plot.plot(Stage, PickList, PickListName, Baseline, Enemy, show_slay_line=ShowSlayLine, multi_target=MultiTarget, ignore_polish=IgnorePolish)
def main(defaults): """ Updates PRONOM signatures Interactive script, requires keyboard input """ try: resume_download = False answers = ['y','yes'] versionXML = defaults['versionXML'].format("{0}","{1}",defaults['containerVersion'],defaults['fidoSignatureVersion'],defaults['version']) #print versionXML print "FIDO signature updater v"+defaults['version'] print "Contacting PRONOM..." currentVersion = getPronomSignature("version") if currentVersion == False: print "Failed to obtain PRONOM signature file version number, please try again" sys.exit() print "Querying latest signaturefile version..." signatureFile = os.path.join(os.path.abspath(defaults['conf_dir']), defaults['signatureFileName'].format(currentVersion)) if os.path.isfile(signatureFile): print "You already have the latest PRONOM signature file, version "+str(currentVersion) ask = raw_input("Update anyway? (yes/no): ") if ask.lower() not in answers: sys.exit() print "Downloading signature file version "+str(currentVersion)+"..." currentFile = getPronomSignature("file") if currentFile == False: print "Failed to obtain PRONOM signature file, please try again" exit() sigfile = open(signatureFile,'wb') sigfile.write(currentFile) sigfile.close() print "Writing {0}...".format(defaults['signatureFileName'].format(currentVersion)) print "Extracting PRONOM PUID's from signature file..." tree = CET.parse(signatureFile) puids = [] for node in tree.iter("{http://www.nationalarchives.gov.uk/pronom/SignatureFile}FileFormat"): puids.append(node.get("PUID")) numberPuids = len(puids) print "Found "+str(numberPuids)+" PRONOM PUID's" print "Downloading signatures can take a while" ask = raw_input("Continue and download signatures? (yes/no): ") if ask.lower() not in answers: print "Aborting update..." sys.exit() tmpdir = os.path.join(os.path.abspath(defaults['conf_dir']), defaults['tmp_dir']) if os.path.isdir(tmpdir): print "Found previously created temporary folder for download:", tmpdir ask = raw_input("Resume download (yes) or start over (no)?: ") if ask.lower() in answers: print "Resuming download..." resume_download = True else: resume_download = False else: print "Creating temporary folder for download:", tmpdir try: os.mkdir(tmpdir) except: pass if not os.path.isdir(tmpdir): tmpdir = os.path.join(os.path.abspath(defaults['conf_dir'])) print "Failed to create temporary folder for PUID's, using", tmpdir print "Downloading signatures, one moment please..." one_percent = (float(numberPuids) / 100) numfiles = 0 for puid in puids: puidType, puidNum = puid.split("/") puidFileName = "puid."+puidType+"."+puidNum+".xml" filename = os.path.join(tmpdir, puidFileName) if os.path.isfile(filename) and checkWellFormedness(filename) and resume_download is not False: numfiles += 1 continue puidUrl = "http://www.nationalarchives.gov.uk/pronom/"+puid+".xml" try: filehandle = urllib.urlopen(puidUrl) except Exception, e: print "Failed to download signaturefile:", puidUrl print "Error:", str(e) print "Please restart and resume download" sys.exit() puidfile = open(filename,'wb') for lines in filehandle.readlines(): puidfile.write(lines) puidfile.close() filehandle.close() if not checkWellFormedness(filename): os.unlink(filename) continue numfiles += 1 percent = int(float(numfiles) / one_percent) print "\r", print str(percent)+"%", time.sleep(defaults['http_throttle']) print "100%" try: import zlib compression = zipfile.ZIP_DEFLATED except: compression = zipfile.ZIP_STORED modes = {zipfile.ZIP_DEFLATED: 'deflated', zipfile.ZIP_STORED: 'stored'} print "Creating PRONOM zip,", zf = zipfile.ZipFile(os.path.join(os.path.abspath(defaults['conf_dir']), defaults['pronomZipFileName'].format(currentVersion)), mode='w') print "adding files with compression mode '"+modes[compression]+"'" for puid in puids: puidType, puidNum = puid.split("/") puidFileName = "puid."+puidType+"."+puidNum+".xml" filename = os.path.join(os.path.abspath(defaults['conf_dir']), defaults['tmp_dir'], puidFileName) if os.path.isfile(filename): zf.write(filename, arcname=puidFileName, compress_type=compression) os.unlink(filename) zf.close() print "Deleting temporary folder and files..." try: for root, dirs, files in os.walk(tmpdir, topdown=False): for name in files: os.remove(os.path.join(root, name)) for name in dirs: os.rmdir(os.path.join(root, name)) os.rmdir(tmpdir) except: pass # update versions.xml versionsFile = os.path.join(os.path.abspath(defaults['conf_dir']), defaults['versionsFileName']) print "Updating {0}...".format(defaults['versionsFileName']) xmlversionsfile = open(versionsFile,'wb') xmlversionsfile.write(versionXML.format(str(currentVersion),"formats-v"+str(currentVersion)+".xml")) xmlversionsfile.close() print "Preparing to convert PRONOM formats to FIDO signatures..." # there should be a check here to handle prepare.main exit() signal (-1/0/1/...) prepare.main() print "FIDO signatures successfully updated" sys.exit()
import tensorflow as tf import numpy as np import prepare, sys X_train, X_lengths, Y_train, dimen, n_steps, X_test, X_test_lengths, y_test, sentences = prepare.main( ) def _last_relevant(output, length): batch_size = tf.shape(output)[0] max_length = int(output.get_shape()[1]) output_size = int(output.get_shape()[2]) index = tf.range(0, batch_size) * max_length + (tf.cast(length, tf.int32) - 1) flat = tf.reshape(output, [-1, output_size]) relevant = tf.gather(flat, index) return relevant def attention(Y, dimen): # [batch_size x seq_len x dim] -- hidden states #Y = tf.constant(np.random.randn(batch_size, seq_len, dim), tf.float32) # [batch_size x dim] -- h_N #h = tf.constant(np.random.randn(batch_size, dim), tf.float32) initializer = tf.random_uniform_initializer() W = tf.get_variable("weights_Y", [dimen, dimen], initializer=initializer) w = tf.get_variable("weights_w", [dimen], initializer=initializer) # [batch_size x seq_len x dim] -- tanh(W^{Y}Y) M = tf.tanh(tf.einsum("aij,jk->aik", Y, W)) # [batch_size x seq_len] -- softmax(Y w^T)
def main(): """ Updates PRONOM signatures """ global defaults print "FIDO signature updater v"+defaults['version'] print "Contacting PRONOM..." currentVersion = getPronomSignature("version") if currentVersion == False: exit() print "Querying latest signaturefile version..." signatureFile = os.path.join(os.path.abspath(defaults['conf_dir']), defaults['signatureFileName'].format(currentVersion)) if os.path.isfile(signatureFile): print "You already have the latest PRONOM signature file, version "+str(currentVersion) answers = {'y','yes'} ask = raw_input("Update anyway? (yes/no): ") if ask.lower() not in answers: sys.exit() print "Downloading signature file version "+str(currentVersion)+"..." currentFile = getPronomSignature("file") if currentFile == False: exit() sigfile = open(signatureFile,'wb') sigfile.write(currentFile) sigfile.close() print "Extracting PRONOM PUID's from signature file..." tree = CET.parse(signatureFile) puids = [] for node in tree.iter("{http://www.nationalarchives.gov.uk/pronom/SignatureFile}FileFormat"): puids.append(node.get("PUID")) numberPuids = len(puids) print "Found "+str(numberPuids)+" PRONOM PUID's" print "Downloading signatures can take a while" answers = {'y','yes'} ask = raw_input("Continue and download signatures? (yes/no): ") if ask.lower() not in answers: print "Aborting update..." sys.exit() tmpdir = os.path.join(os.path.abspath(defaults['conf_dir']), defaults['tmp_dir']) print "Creating temporary folder for download:", tmpdir try: os.mkdir(tmpdir) except: pass if not os.path.isdir(tmpdir): tmpdir = os.path.join(os.path.abspath(defaults['conf_dir'])) print "Failed to create temporary folder for PUID's, using", tmpdir print "Downloading signatures, one moment please..." one_percent = (float(numberPuids) / 100) numfiles = 0 for puid in puids: puidType, puidNum = puid.split("/") filehandle = urllib.urlopen('http://www.nationalarchives.gov.uk/pronom/'+puid+'.xml') puidFileName = "puid."+puidType+"."+puidNum+".xml" filename = os.path.join(os.path.abspath(defaults['conf_dir']), defaults['tmp_dir'], puidFileName) puidfile = open(filename,'wb') for lines in filehandle.readlines(): puidfile.write(lines) puidfile.close() filehandle.close() if not checkWellFormedness(filename): os.unlink(filename) continue numfiles += 1 percent = int(float(numfiles) / one_percent) print "\b\b\b\b\b", print str(percent)+"%", time.sleep(defaults['http_throttle']) print "\b\b\b\b\b", print "100%" try: import zlib compression = zipfile.ZIP_DEFLATED except: compression = zipfile.ZIP_STORED modes = {zipfile.ZIP_DEFLATED: 'deflated', zipfile.ZIP_STORED: 'stored'} print "Creating PRONOM zip,", zf = zipfile.ZipFile(os.path.join(os.path.abspath(defaults['conf_dir']), defaults['pronomZipFile']), mode='w') print "adding files with compression mode '"+modes[compression]+"'" for puid in puids: puidType, puidNum = puid.split("/") puidFileName = "puid."+puidType+"."+puidNum+".xml" filename = os.path.join(os.path.abspath(defaults['conf_dir']), defaults['tmp_dir'], puidFileName) if os.path.isfile(filename): zf.write(filename, arcname=puidFileName, compress_type=compression) os.unlink(filename) zf.close() print "Deleting temporary folder and files..." try: for root, dirs, files in os.walk(tmpdir, topdown=False): for name in files: os.remove(os.path.join(root, name)) for name in dirs: os.rmdir(os.path.join(root, name)) os.rmdir(tmpdir) except: pass print "Preparing to convert PRONOM formats to FIDO signatures..." prepare.main() print "FIDO signatures successfully updated" sys.exit()
def main(defaults): """ Updates PRONOM signatures Interactive script, requires keyboard input """ try: resume_download = False answers = ['y','yes'] versionXML = defaults['versionXML'].format("{0}","{1}",defaults['containerVersion'],defaults['fidoSignatureVersion'],defaults['version']) #print versionXML print "FIDO signature updater v"+defaults['version'] print "Contacting PRONOM..." currentVersion = getPronomSignature("version") if currentVersion == False: print "Failed to obtain PRONOM signature file version number, please try again" sys.exit() print "Querying latest signaturefile version..." signatureFile = os.path.join(os.path.abspath(defaults['conf_dir']), defaults['signatureFileName'].format(currentVersion)) versionsFile = os.path.join(os.path.abspath(defaults['conf_dir']), defaults['versionsFileName']) if os.path.isfile(signatureFile): print "You already have the latest PRONOM signature file, version "+str(currentVersion) ask = raw_input("Update anyway? (yes/no): ") if ask.lower() not in answers: sys.exit() print "Updating {0}...".format(defaults['versionsFileName']) xmlversionsfile = open(versionsFile,'wb') xmlversionsfile.write(versionXML.format(str(currentVersion),"formats-v"+str(currentVersion)+".xml")) xmlversionsfile.close() print "Downloading signature file version "+str(currentVersion)+"..." currentFile = getPronomSignature("file") if currentFile == False: print "Failed to obtain PRONOM signature file, please try again" exit() sigfile = open(signatureFile,'wb') sigfile.write(currentFile) sigfile.close() print "Writing {0}...".format(defaults['signatureFileName'].format(currentVersion)) print "Extracting PRONOM PUID's from signature file..." tree = CET.parse(signatureFile) puids = [] for node in tree.iter("{http://www.nationalarchives.gov.uk/pronom/SignatureFile}FileFormat"): puids.append(node.get("PUID")) numberPuids = len(puids) print "Found "+str(numberPuids)+" PRONOM PUID's" print "Downloading signatures can take a while" ask = raw_input("Continue and download signatures? (yes/no): ") if ask.lower() not in answers: print "Aborting update..." sys.exit() tmpdir = os.path.join(os.path.abspath(defaults['conf_dir']), defaults['tmp_dir']) if os.path.isdir(tmpdir): print "Found previously created temporary folder for download:", tmpdir ask = raw_input("Resume download (yes) or start over (no)?: ") if ask.lower() in answers: print "Resuming download..." resume_download = True else: resume_download = False else: print "Creating temporary folder for download:", tmpdir try: os.mkdir(tmpdir) except: pass if not os.path.isdir(tmpdir): tmpdir = os.path.join(os.path.abspath(defaults['conf_dir'])) print "Failed to create temporary folder for PUID's, using", tmpdir print "Downloading signatures, one moment please..." one_percent = (float(numberPuids) / 100) numfiles = 0 for puid in puids: puidType, puidNum = puid.split("/") puidFileName = "puid."+puidType+"."+puidNum+".xml" filename = os.path.join(tmpdir, puidFileName) if os.path.isfile(filename) and checkWellFormedness(filename) and resume_download is not False: numfiles += 1 continue puidUrl = "http://www.nationalarchives.gov.uk/pronom/"+puid+".xml" try: filehandle = urllib.urlopen(puidUrl) except Exception, e: print "Failed to download signaturefile:", puidUrl print "Error:", str(e) print "Please restart and resume download" sys.exit() puidfile = open(filename,'wb') for lines in filehandle.readlines(): puidfile.write(lines) puidfile.close() filehandle.close() if not checkWellFormedness(filename): os.unlink(filename) continue numfiles += 1 percent = int(float(numfiles) / one_percent) print "\b\b\b\b\b", print str(percent)+"%", time.sleep(defaults['http_throttle']) print "\b\b\b\b\b", print "100%" try: import zlib compression = zipfile.ZIP_DEFLATED except: compression = zipfile.ZIP_STORED modes = {zipfile.ZIP_DEFLATED: 'deflated', zipfile.ZIP_STORED: 'stored'} print "Creating PRONOM zip,", zf = zipfile.ZipFile(os.path.join(os.path.abspath(defaults['conf_dir']), defaults['pronomZipFileName'].format(currentVersion)), mode='w') print "adding files with compression mode '"+modes[compression]+"'" for puid in puids: puidType, puidNum = puid.split("/") puidFileName = "puid."+puidType+"."+puidNum+".xml" filename = os.path.join(os.path.abspath(defaults['conf_dir']), defaults['tmp_dir'], puidFileName) if os.path.isfile(filename): zf.write(filename, arcname=puidFileName, compress_type=compression) os.unlink(filename) zf.close() print "Deleting temporary folder and files..." try: for root, dirs, files in os.walk(tmpdir, topdown=False): for name in files: os.remove(os.path.join(root, name)) for name in dirs: os.rmdir(os.path.join(root, name)) os.rmdir(tmpdir) except: pass print "Preparing to convert PRONOM formats to FIDO signatures..." prepare.main() print "FIDO signatures successfully updated" sys.exit()
# Prepare # ================================= """ This module performs several steps in preparing the data for analysis. First, it splits each text into segments of a given length. Second, it either takes all segments or samples an equal number of segments per text. Third, it selects the desired features from each segment (form and pos) Fourth, it creates document-term matrixes with absolute, relative and binary feature counts. This function needs to be run again when a parameter is changed. """ segmentlength = 3000 max_num_segments = -1 featuretype = ["lemmata", "NOM"] # forms, pos absolutefreqs, relativefreqs, binaryfreqs = prepare.main(taggedfolder, segmentfolder, datafolder, dtmfolder, segmentlength, max_num_segments, stoplistfile, featuretype) # ================================= # Calculate # ================================= """ This module performs the actual distinctiveness measure for each feature. The calculation can be based on relative or binary features. The calculation can work in several ways: by division, subtraction as well as with or without applying some log transformation. The contrast parameter takes ["category", "group1", "group2"] as in the metadata table. """ separator = ";" contrast = ["group", "Racine", "contemporaries"] # example for roman20
def all_in_one(config): prepare.main(config) calc_data.main(config) calc_data.calc_overall_configured(config) plot_dets.main(config)
def prepare_experiment(experiment_config): for cfg in experiment_config: logging.info("Preparing models to '%s'", cfg.MODELS_DIR) prepare.main(cfg)