def run_agree_disagree_sim(language, viol, mgain, nconstraints, mb, gam,
parameters, reducemem):
'''
this is not described anywhere, is work in progress
makes a bunch of constraints on the basis of the natural classes structure of the language
they have the form +f +f, +f [+wb] +f, +f [-wb] +f, and ditto for every combination of + and - for f
(in other words, it makes agree constraints and disagree constraints for every feature and nat class in the language)
it then runs a simulation with this premade constraint set.
the early results in testing this have not been encouraging
'''
basepath = os.getcwd().split('code')[0]
lgfullpath = os.path.join(basepath, 'data', language)
simfunc.cleanUpWorkdir(basepath)
simfunc.makeSimFiles(lgfullpath, ag_disag=True)
if parameters:
params.move_params(lgfullpath, 'params.txt')
viol, mgain, nconstraints, gam = params.read_params()
else:
params.makeParams(consize=nconstraints,
violable=viol,
mingain=mgain,
gamma=gam,
ag_disag=True)
simfunc.runBaselineSim(basepath, reducemem=reducemem)
wrapstring = os.path.join(
'sims', '_'.join([
language.replace(os.sep, "_"), 'baseline', 'AG', viol[:2], 'gain',
mingain, 'ncons', nconstraints
]))
return simfunc.wrapSims(wrapstring, ret=True)
def run_baseline_sim(language, viol, mgain, nconstraints, mb, gam, parameters,
reducemem):
'''
this function runs the baseline simulation with a default (segmental) projection
if it does not succeed, it does not fail gracefully, so be forwarned
'''
basepath = os.getcwd().split('code')[0]
lgfullpath = os.path.join(basepath, 'data', language)
simfunc.cleanUpWorkdir(basepath)
if parameters:
params.move_params(os.path.join(lgfullpath, 'params.txt'))
viol, mgain, nconstraints, gam = params.read_params()
else:
params.makeParams(consize=nconstraints,
violable=viol,
mingain=mgain,
gamma=gam,
predefault=False)
simfunc.makeSimFiles(lgfullpath)
try:
simfunc.runBaselineSim(basepath, reducemem=reducemem)
#language=language.split('../data/')[1].replace('/','_')
wrapstring = os.path.join(
'sims',
language.replace(os.sep, "_") + '_baseline' + '_gain' + mgain +
'_con' + nconstraints)
simfunc.wrapSims(wrapstring)
except CalledProcessError:
print("Done")
def run_wb_sim(language, viol, mgain, nconstraints, mb, gam, parameters,
reducemem):
'''
this learning simulation is described in Gouskova and Gallagher (NLLT). The learner starts with a baseline
grammar; if this grammar contains placeholder trigrams, it creates projections for each distinct trigram and runs a final simulation with those projections available.
'''
basepath = os.getcwd().split('code')[0]
simfunc.cleanUpWorkdir(basepath)
if parameters:
params.move_params(
os.path.join(basepath, 'data', language, 'params.txt'))
viol, mgain, nconstraints, gamma = params.read_params()
else:
params.makeParams(consize=nconstraints,
violable=viol,
mingain=mgain,
gamma=gam,
predefault=False)
simfunc.makeSimFiles(language)
#baseline simulation
simfunc.runBaselineSim(basepath, reducemem=reducemem)
#analyze resulting grammar.txt file, make projections for each wb-mentioning constraint
simfunc.makeProjection(basepath, 'wb', mb)
if len(os.listdir('projections')) == 0:
print(
'\nNo projections were found because there were no placeholder constraints in the baseline grammar.'
)
else:
simfunc.runCustomSim(reducemem=reducemem, simtype='wb')
vio = viol[0:2]
wrapstring = os.path.join('sims', language.replace(
os.sep, "_")) + "_" + '_'.join(
['wb', vio, 'gain' + str(mgain), 'con' + str(nconstraints)])
return simfunc.wrapSims(wrapstring, ret=True)
def __init__(self, use_gpu=None, enable_mkldnn=None):
"""
initialize with the necessary elements
"""
cfg = read_params()
if use_gpu is not None:
cfg.use_gpu = use_gpu
if enable_mkldnn is not None:
cfg.enable_mkldnn = enable_mkldnn
cfg.hubserving = True
cfg.enable_benchmark = False
self.args = cfg
if cfg.use_gpu:
try:
_places = os.environ["CUDA_VISIBLE_DEVICES"]
int(_places[0])
print("Use GPU, GPU Memery:{}".format(cfg.gpu_mem))
print("CUDA_VISIBLE_DEVICES: ", _places)
except:
raise RuntimeError(
"Environment Variable CUDA_VISIBLE_DEVICES is not set correctly. If you wanna use gpu, please set CUDA_VISIBLE_DEVICES via export CUDA_VISIBLE_DEVICES=cuda_device_id."
)
else:
print("Use CPU")
print("Enable MKL-DNN") if enable_mkldnn else None
self.predictor = create_paddle_predictor(self.args)
def add_param_columns(df, scaffold=False):
for job_file, job_df in df.groupby(level=0):
if True: # try to parse it as a GAN
job_params = params.read_params(job_file, line_start='# ')
data_model_file = os.path.join(
os.path.dirname(job_file), job_params['model_dir'],
job_params['data_model_name'] + '.model')
data_model_params = params.read_params(data_model_file,
line_start='# ')
job_params.update(
prepend_keys(data_model_params, prefix='data_model_params.'))
gen_model_file = os.path.join(
os.path.dirname(job_file), job_params['model_dir'],
job_params['gen_model_name'] + '.model')
gen_model_params = params.read_params(gen_model_file,
line_start='# ')
if scaffold:
for k, v in models.scaffold_model(gen_model_file).items():
df.loc[job_file, 'gen_' + k] = v
job_params.update(
prepend_keys(gen_model_params, prefix='gen_model_params.'))
disc_model_file = os.path.join(
os.path.dirname(job_file), job_params['model_dir'],
job_params['disc_model_name'] + '.model')
disc_model_params = params.read_params(disc_model_file,
line_start='# ')
if scaffold:
for k, v in models.scaffold_model(disc_model_file).items():
df.loc[job_file, 'disc_' + k] = v
job_params.update(
prepend_keys(disc_model_params, prefix='disc_model_params.'))
del job_params['seed'] # these already exist
del job_params['fold']
try:
pass
except AttributeError:
try: # try to parse model name as a GAN
model_params = models.parse_gan_name(model_name)
except AttributeError:
model_params = models.parse_gen_name(model_name)
for param, value in job_params.items():
df.loc[job_file, param] = value
return job_params
def scaffold_model(model_file, force=True):
scaff_out_file = model_file + '.scaffold_output'
try:
assert not force
scaff_params = params.read_params(scaff_out_file)
print(scaff_out_file)
except (IOError, AssertionError):
net = caffe_util.Net(model_file, phase=caffe.TRAIN)
scaff_params = params.Params()
scaff_params['n_params'] = net.get_n_params()
scaff_params['n_activs'] = net.get_n_activs()
scaff_params['size'] = net.get_approx_size()
scaff_params['min_width'] = net.get_min_width()
params.write_params(scaff_out_file, scaff_params)
return scaff_params
def _calc_rate(mode, run_nr, threshold=None, data_folder=data_folder_default):
if mode == "hit":
read_scores = read_hitscores
threshold_name = 'hitscoreThreshold'
elif mode == "mul":
read_scores = read_mulscores
threshold_name = 'multiscoreThreshold'
else:
print "ERROR: illegal mode %s" % mode
return
scores = read_scores(run_nr=run_nr, ol=1, data_folder=data_folder)
if scores is None:
return None
else:
n_frames = len(scores)
if threshold is None:
T = params.read_params(params_fn, run_nr)[threshold_name]
else:
T = threshold
n_hits = (scores > T).sum()
return (float(n_hits) / float(n_frames))
def run_handmade_projection_sim(language, viol, mgain, nconstraints, gam,
parameters, feature, reducemem):
'''
this either creates a projection file based on the value of "feature" (e.g., "-son") or runs a simulation with a custom projection file. To do the latter, supply a full path to the projection file as the last argument.
'''
basepath = os.getcwd().split('code')[0]
lgfullpath = os.path.join(basepath, 'data', language)
simfunc.cleanUpWorkdir(basepath)
simfunc.makeSimFiles(language)
if parameters:
params.move_params(lgfullpath, 'params.txt')
viol, mgain, nconstraints, gam = params.read_params()
else:
params.makeParams(consize=nconstraints,
violable=viol,
mingain=mgain,
gamma=gam)
simfunc.handmakeProjection(basepath, feature)
simfunc.runCustomSim(feature, reducemem=reducemem)
if 'output_baseline' in os.listdir(basepath):
shutil.rmtree(basepath + 'output_baseline')
simfunc.wrapSims(os.path.join(
'sims', '_'.join(language.replace(os.sep, "_"), 'custom')),
cust=True)
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from paddle_serving_client import Client
import cv2
import sys
import numpy as np
import os
import time
import re
import base64
from tools.infer.predict_cls import TextClassifier
from params import read_params
global_args = read_params()
if global_args.use_gpu:
from paddle_serving_server_gpu.web_service import WebService
else:
from paddle_serving_server.web_service import WebService
class TextClassifierHelper(TextClassifier):
def __init__(self, args):
self.cls_image_shape = [
int(v) for v in args.cls_image_shape.split(",")
]
self.cls_batch_num = args.rec_batch_num
self.label_list = args.label_list
self.cls_thresh = args.cls_thresh
self.fetch = [
add_config_file_argument('--nr-frames',
type=int,
help='Number of frames',
default=None)
add_config_file_argument('--skip-tof', action='store_true')
add_config_file_argument('--only-save-multiples', action='store_true')
args = argparser.parse_args()
this_dir = os.path.dirname(os.path.realpath(__file__))
sys.path.append(this_dir)
from conf import *
# Read in parameters from a csv file
import params
p = params.read_params('params.csv', args.run_nr)
gap_top = p['pnccdGapTopMM'] * 1E-3
gap_bottom = p['pnccdGapBottomMM'] * 1E-3
gap_total = gap_top + gap_bottom
center_shift = int((gap_top - gap_bottom) / pixel_size)
# Dark file
if args.dark_nr:
darkfile_nr = args.dark_nr
else:
darkfile_nr = p['darkNr']
# Hitscore threshold
if args.hitscore_threshold is not None:
hitScoreThreshold = args.hitscore_threshold
else:
def main():
# Add arg types
parser = argparse.ArgumentParser()
parser.add_argument("-s",
"--simple",
help="Don't display graphics",
action="store_true")
parser.add_argument("-d",
"--delay",
help="Delay between turns",
type=float)
parser.add_argument("-p",
"--parameters_file",
help="Load parameters from the given file")
# Parse args
args = parser.parse_args()
delay = 0
if args.delay:
delay = args.delay
if args.parameters_file:
params.read_params(args.parameters_file)
# Start the game
renderer, world = init_game()
turn = 0
population = [0 for _ in range(params.MAX_LIFE)]
done = False
while not done:
# Get input
events = pygame.event.get()
for event in events:
if event.type == pygame.QUIT:
done = True
# Prepare the next turn
world.turn_start()
# Update the world
world.update()
# Complete the current turn
world.turn_end()
# Draw world
if not args.simple:
renderer.draw_world(world)
# Update population buffer
current = len(world.blips.keys())
population[turn % params.MAX_LIFE] = current
if turn >= params.MAX_LIFE - 1:
# Compute statistic for the period
avg = sum(population) / len(population)
best = max(population)
worst = min(population)
# Check if population has stabilized
eps = 0.1 * avg
if abs(best - avg) < eps and abs(worst - avg) < eps:
done = True
# Print some results in the terminal
print("Best: {0}; Worst: {1}, Avg: {2}, Current {3}".format(
best, worst, avg, current))
turn += 1
if current == 0:
done = True
# Time between rounds
time.sleep(delay)
pygame.quit()
def run_mbsublex_sim(language, viol, mgain, nconstraints, mb, gam, parameters,
reducemem):
'''
this simulation needs a corpus with morpheme boundaries.
it starts by running a baseline simulation on the corpus.
then, if it finds constraints in the resulting grammar that mention [-mb] (see mbsublex module), it splits the learning data into individual morphemes, one morph per line
it then uses that as a new baseline data set. if it locates any placeholder trigrams in that subset, it makes a projection from them
and then it runs a projection simulation on the morpheme sublexicon, and on the whole corpus.
'''
basepath = os.getcwd().split('code')[0]
maxentdir = os.path.join(basepath, 'maxent2', 'temp')
dircontent = os.listdir(maxentdir)
vio = viol[0:2]
wrapstring = os.path.join('sims', language.replace(
os.sep, "_")) + "_" + '_'.join([
'wb', 'mbsublex', vio, 'gain' + str(mgain),
'con' + str(nconstraints)
])
if not 'output_baseline' in dircontent:
simfunc.cleanUpWorkdir(basepath)
if parameters:
params.move_params(
os.path.join(basepath, 'data', language, 'params.txt'))
viol, mgain, nconstraints, gamma = params.read_params()
else:
params.makeParams(consize=nconstraints,
violable=viol,
mingain=mgain,
gamma=gam,
predefault=False)
simfunc.makeSimFiles(language)
print('running the baseline simulation using original training corpus')
simfunc.runBaselineSim(
basepath, reducemem=reducemem, rt_output_baseline=False
) #copies grammar, proj, tableau, maxentouptut in maxent2/temp/output_baseline
mbsublex.move_sublex_files(kind='output_baseline')
if not 'output_mbsublex_baseline' in dircontent:
print("Baseline simulation found at " +
os.path.join(maxentdir, 'output_baseline'))
#analyze resulting grammar.txt file for [-mb] constraints, and make projections
found_mb = mbsublex.search_grammar_for_mb()
if found_mb:
print('Making a sublexicon with one morph per line')
mbsublex.make_freewd_sublexicon(
) #renames curr corpus 'orig_corpus.txt', creates new corpus that consists of just morphologically simple words and is called 'corpus.txt'
print(
'Running a new baseline simulation using a sublexicon as training data'
)
if parameters:
params.scale_params(inpath=os.path.join(
basepath, 'data', language, 'params.txt'),
multiply_by=0.01,
keepconsize=True)
else:
params.scale_params(viol, gain, consize, gamma, 0.01,
True) #last one is keepconsize
simfunc.runBaselineSim(basepath,
reducemem=reducemem,
rt_output_baseline=False)
mbsublex.move_sublex_files(kind="output_mbsublex_baseline")
else:
print('Did not find any *X-mb-Y trigrams. Quitting now.')
return mbsublex.wrapSims(wrapstring, basepath, ret=True)
if not 'output_mbsublex' in dircontent:
print("Sublexicon baseline simulation found at " +
os.path.join(maxentdir, 'output_mbsublex'))
mbsublex.makeProjection(basepath, 'wb', mb=True)
print(
'projections found--running a projection simulation on morph sublexicon'
)
simfunc.runCustomSim(reducemem=reducemem, simtype='wb')
mbsublex.move_sublex_files(kind='output_mbsublex')
if not 'output_final' in dircontent:
mbsublex.rename_corpus_back()
print(
'now running a projection simulation on the original training corpus'
)
if parameters:
params.scale_params(inpath=os.path.join(basepath, 'data', language,
'params.txt'),
multiply_by=1,
keepconsize=True)
else:
params.scale_params(viol, gain, consize, gamma, 10, True)
simfunc.runCustomSim(reducemem=reducemem, simtype='wb')
mbsublex.move_sublex_files(kind='output_final')
print('done!')
try:
return mbsublex.wrapSims(wrapstring, basepath=maxentdir, ret=True)
except:
print("The simulation failed for some reason. Check the contents of " +
maxentdir + " to help with debugging.")
