def optimizeEps():
pdict, fname = setParams()
freqlist = [5, 10, 15, 20, 25]
freqlist.extend(range(35, 310, 25))
epslist = [k * pdict['circrad'] for k in np.arange(0.05, 1.8, 0.05)]
u_exact, v_exact, u_negex, v_negex, w_negex, uz_negex, vz_negex, wz_negex, u_gauss, v_gauss, w_gauss, uz_gauss, vz_gauss, wz_gauss = varyFreqEps_zVel(
pdict, freqlist, epslist)
mydict = {}
mydict['u_exact'] = u_exact
mydict['v_exact'] = v_exact
mydict['u_negex'] = u_negex
mydict['v_negex'] = v_negex
mydict['w_negex'] = w_negex
mydict['u_gauss'] = u_gauss
mydict['v_gauss'] = v_gauss
mydict['w_gauss'] = w_gauss
mydict['uz_negex'] = uz_negex
mydict['vz_negex'] = vz_negex
mydict['wz_negex'] = wz_negex
mydict['uz_gauss'] = uz_gauss
mydict['vz_gauss'] = vz_gauss
mydict['wz_gauss'] = wz_gauss
mydict['pdict'] = pdict
mydict['freqlist'] = freqlist
mydict['epslist'] = epslist
F = open(fname + '.pickle', 'w')
Pickler(F).dump(mydict)
F.close()
adderrs(fname, mydict)
def pickle_results(self):
data = file('participant_%d.dat' % self.participant_id,'w')
p = Pickler(data)
experiment = {}
experiment['test_sentences'] = self.test_sentences
experiment['results'] = self.results
p.dump(experiment)
def pickle_matrices(matrices, outdir='.'): """Pickles dictionary of matrices output by create_matrices""" for name, matrix in matrices.iteritems(): fpath = os.path.join(outdir, name + '.pickle') with open(fpath, 'wb') as fh: pickler = Pickler(fh, HIGHEST_PROTOCOL) pickler.dump(matrix)
def replicate(self, target):
d = {'id': self.id,
'user': self.user,
'description': self.description,
'entries': self._entries}
f = StringIO()
p = Pickler(f)
p.dump(d)
payloadStr = f.getvalue()
LOG('Replication', INFO, 'replicate> transaction id: %s; '
'size (uncompressed): %s' % (
oid2str(self.id), len(payloadStr))) #DBG
payloadStr = compress(payloadStr)
handler = FixedHTTPHandler()
opener = urllib2.build_opener(handler)
urllib2.install_opener(opener)
LOG('Replication', INFO, 'replicate> transaction id: %s; size: %s' % (
oid2str(self.id), len(payloadStr))) #DBG
url = '%s/load' % target.url
schema, domain, path, x1, x2, x3 = urlparse.urlparse(url)
newurl = '%s://%s:%s@%s%s' % (
schema, target.username, target.password, domain, path)
try:
urllib2.urlopen(newurl, urllib.urlencode({'data': payloadStr}))
except urllib2.HTTPError, e:
if e.code != 204: # 204 == 'No content' which is what we expect
raise
def __setitem__(self, key, value):
if self.writeback:
self.cache[key] = value
f = StringIO()
p = Pickler(f, self._protocol)
p.dump(value)
self.dict[key] = f.getvalue()
def cloneByPickle(obj, ignore_list=()):
"""Makes a copy of a ZODB object, loading ghosts as needed.
Ignores specified objects along the way, replacing them with None
in the copy.
"""
ignore_dict = {}
for o in ignore_list:
ignore_dict[id(o)] = o
def persistent_id(ob, ignore_dict=ignore_dict):
if ignore_dict.has_key(id(ob)):
return 'ignored'
if getattr(ob, '_p_changed', 0) is None:
ob._p_changed = 0
return None
def persistent_load(ref):
assert ref == 'ignored'
# Return a placeholder object that will be replaced by
# removeNonVersionedData().
placeholder = SimpleItem()
placeholder.id = "ignored_subobject"
return placeholder
stream = StringIO()
p = Pickler(stream, 1)
p.persistent_id = persistent_id
p.dump(obj)
stream.seek(0)
u = Unpickler(stream)
u.persistent_load = persistent_load
return u.load()
def _pickled_setitem_(self, key, value):
""" Add object (pickle if needed) to dbase
>>> db['A/B/C'] = obj
"""
##
if self.writeback:
self.cache[key] = value
## not TObject? pickle it and convert to Ostap.BLOB
if not isinstance(value, ROOT.TObject):
## (1) pickle it
f = BytesIO()
p = Pickler(f, self.protocol)
p.dump(value)
## (2) zip it
z = zlib.compress(f.getvalue(), self.compresslevel)
## (3) put it into BLOB
from ostap.core.core import Ostap
blob = Ostap.BLOB(key)
status = Ostap.blob_from_bytes(blob, z)
value = blob
del z, f, p
## finally use ROOT
self.dict[key] = value
def pickle_self(self):
ofile = open(
join(self.p.picklepath,
self.ownname + '_' + self.p.label + '.txt'), 'w')
einmachglas = Pickler(ofile)
einmachglas.dump(self)
ofile.close()
def test_config_and_collector_pickling(self, testdir):
from cPickle import Pickler, Unpickler
tmpdir = testdir.tmpdir
dir1 = tmpdir.ensure("somedir", dir=1)
config = testdir.parseconfig()
col = config.getfsnode(config.topdir)
col1 = col.join(dir1.basename)
assert col1.parent is col
io = py.std.cStringIO.StringIO()
pickler = Pickler(io)
pickler.dump(col)
pickler.dump(col1)
pickler.dump(col)
io.seek(0)
unpickler = Unpickler(io)
topdir = tmpdir.ensure("newtopdir", dir=1)
topdir.ensure("somedir", dir=1)
old = topdir.chdir()
try:
newcol = unpickler.load()
newcol2 = unpickler.load()
newcol3 = unpickler.load()
assert newcol2.config is newcol.config
assert newcol2.parent == newcol
assert newcol2.config.topdir.realpath() == topdir.realpath()
assert newcol.fspath.realpath() == topdir.realpath()
assert newcol2.fspath.basename == dir1.basename
assert newcol2.fspath.relto(newcol2.config.topdir)
finally:
old.chdir()
def tryToResolveConflict(self,
oid,
committedSerial,
oldSerial,
newpickle,
committedData=''):
# class_tuple, old, committed, newstate = ('',''), 0, 0, 0
try:
prfactory = PersistentReferenceFactory()
newpickle = self._crs_untransform_record_data(newpickle)
file = StringIO(newpickle)
unpickler = Unpickler(file)
unpickler.find_global = find_global
unpickler.persistent_load = prfactory.persistent_load
meta = unpickler.load()
if isinstance(meta, tuple):
klass = meta[0]
newargs = meta[1] or ()
if isinstance(klass, tuple):
klass = find_global(*klass)
else:
klass = meta
newargs = ()
if klass in _unresolvable:
raise ConflictError
newstate = unpickler.load()
inst = klass.__new__(klass, *newargs)
try:
resolve = inst._p_resolveConflict
except AttributeError:
_unresolvable[klass] = 1
raise ConflictError
old = state(self, oid, oldSerial, prfactory)
committed = state(self, oid, committedSerial, prfactory, committedData)
resolved = resolve(old, committed, newstate)
file = StringIO()
pickler = Pickler(file, 1)
pickler.inst_persistent_id = persistent_id
pickler.dump(meta)
pickler.dump(resolved)
return self._crs_transform_record_data(file.getvalue(1))
except (ConflictError, BadClassName):
pass
except:
# If anything else went wrong, catch it here and avoid passing an
# arbitrary exception back to the client. The error here will mask
# the original ConflictError. A client can recover from a
# ConflictError, but not necessarily from other errors. But log
# the error so that any problems can be fixed.
logger.error("Unexpected error", exc_info=True)
raise ConflictError(oid=oid,
serials=(committedSerial, oldSerial),
data=newpickle)
def deepCopy(obj):
stream = StringIO()
p = Pickler(stream, 1)
p.dump(obj)
stream.seek(0)
u = Unpickler(stream)
return u.load()
def serialize(self, obj, deferred=False):
"""Serialize an object
:param obj: The object to serialize.
:param deferred: When this is true Deferred objects are
serialized and their values are loaded on deserialization.
When this is false Deferred objects are not serializable.
"""
if deferred:
args = {}
def persistent_id(obj):
if isinstance(obj, Deferred):
args[obj.id] = obj
return obj.id
return None
else:
args = None
def persistent_id(obj):
if isinstance(obj, Deferred):
raise PicklingError('%s cannot be serialized' % obj)
return None
data = StringIO()
pickle = Pickler(data, HIGHEST_PROTOCOL)
pickle.persistent_id = persistent_id
pickle.dump(obj)
msg = data.getvalue()
return (msg, args) if deferred else msg
def validate(self):
if self.opt.verbose:
print(self.testname)
if not os.path.exists(self.cp.get("jing", "path")):
print("Error: jing not found.")
print(" Looked in: %s" % self.cp.get("jing", "path"))
sys.exit()
m = re.match("(?sm).*version=\"([.0-9a-z]+)\".*", self.data["csl"])
if m:
rnc_path = os.path.join(self.cp.get("csl", "v%s" % m.group(1)))
else:
print("Error: Unable to find CSL version in %s" % self.hp)
sys.exit()
tfd, tfilename = tempfile.mkstemp(dir=".")
os.write(tfd, self.data["csl"].encode('utf8'))
os.close(tfd)
jfh = os.popen("%s %s -c %s %s" %
(self.cp.get("jing", "command"),
self.cp.get("jing", "path"), rnc_path, tfilename))
success = True
plural = ""
while 1:
line = jfh.readline()
if not line: break
line = line.strip()
e = re.match("^fatal:", line)
if e:
print(line)
sys.exit()
m = re.match(".*:([0-9]+):([0-9]+): *error:(.*)", line)
if m:
if success:
print("\n##")
print("#### Error%s in CSL for test: %s" %
(plural, self.hp))
print("##\n")
success = False
print(" %s @ line %s" % (m.group(3).upper(), m.group(1)))
plural = "s"
jfh.close()
os.unlink(tfilename)
if not success:
print("")
io = StringIO()
io.write(self.data["csl"])
io.seek(0)
linepos = 1
while 1:
cslline = io.readline()
if not cslline: break
cslline = cslline.rstrip()
print("%3d %s" % (linepos, cslline))
linepos += 1
pfh = open(self.pickle, "wb+")
pickler = Pickler(pfh)
pickler.dump((opt, self.pos))
sys.exit()
def pickle_the_memory(self):
outfile = open(
join(
self.p.datapath, self.ea.ownname + '_cecLog_memory_' +
self.p.label[:-5] + '.txt'), 'w')
einmachglas = Pickler(outfile)
einmachglas.dump(self.memory)
outfile.close()
def save(self, model_path=None):
"""
Save the classifier model to disk in pickle format
:param model_path: path to save models
:type model_path: str
"""
path = 'models/classifiers.m' if model_path is None else model_path
if self.classifiers:
with open(path, 'w') as f:
pickle = Pickler(f, -1)
pickle.dump(self.classifiers)
if self.vectorizer:
with open(path.replace('.m', '.vec'), 'w') as f:
pickle = Pickler(f, -1)
pickle.dump(self.vectorizer)
def compress_item ( self , value ) :
"""Compress (zip) the item using ``bz2.compress''
- see bz2.compress
"""
f = BytesIO ()
p = Pickler ( f , self.protocol )
p.dump ( value )
return bz2.compress ( f.getvalue() , self.compresslevel )
def validate(self):
if self.opt.verbose:
print self.testname
if not os.path.exists(os.path.join("..", "jing")):
print "Error: jing not found as sibling of processor archive."
print " Looked in: %s" % os.path.join("..", "jing")
sys.exit()
m = re.match("(?sm).*version=\"([.0-9a-z]+)\".*", self.data["csl"])
if m:
rnc_path = os.path.join("csl", "%s" % m.group(1), "csl.rnc")
else:
print "Error: Unable to find CSL version in %s" % self.hp
sys.exit()
tfd, tfilename = tempfile.mkstemp(dir=".")
os.write(tfd, self.data["csl"])
os.close(tfd)
jfh = os.popen("java -jar %s -c %s %s" % (os.path.join(
"..", "jing", "bin", "jing.jar"), rnc_path, tfilename))
success = True
plural = ""
while 1:
line = jfh.readline()
if not line: break
line = line.strip()
e = re.match("^fatal:", line)
if e:
print line
sys.exit()
m = re.match(".*:([0-9]+):([0-9]+): *error:(.*)", line)
if m:
if success:
print "\n##"
print "#### Error%s in CSL for test: %s" % (plural,
self.hp)
print "##\n"
success = False
print " %s @ line %s" % (m.group(3).upper(), m.group(1))
plural = "s"
jfh.close()
os.unlink(tfilename)
if not success:
print ""
io = StringIO()
io.write(self.data["csl"])
io.seek(0)
linepos = 1
while 1:
cslline = io.readline()
if not cslline: break
cslline = cslline.rstrip()
print "%3d %s" % (linepos, cslline)
linepos += 1
pfh = open(self.pickle, "w+b")
pickler = Pickler(pfh)
pickler.dump((opt, self.pos))
sys.exit()
def checkXLine_multfreqs(basedir,
basename,
freqlist=[10 * k for k in range(1, 31)]):
'''
Calculates motion along the x-axis caused by a sphere centered at the origin oscillating as the sum of sinusoids of all the frequencies in freqlist simultaneously. Also handles lists of one element - single frequencies.
Uses Stokeslet+dipole soln, Fourier and quasi-steady methods.
'''
a = 1.0
mu = 1.0
rho = 1.0
U = np.array([1.0 / len(freqlist), 0.0, 0.0])
c = np.array([0.0, 0.0, 0.0])
x = np.linspace(a, a * 10.0, 100)
y = np.zeros(x.shape)
z = np.zeros(x.shape)
pdict = {
'a': a,
'mu': mu,
'borderinit': np.array([x[0], y[0], z[0]]),
'centerinit': c,
'freqlist': freqlist,
'U': U
}
period = 1.0 / min(freqlist)
T = 10.0 * period
dt = period / 40.0
tvec = np.arange(0, T + dt, dt)
mydict = {
'u_fourier': 0.0,
'v_fourier': 0.0,
'w_fourier': 0.0,
'u_quasi': 0.0,
'v_quasi': 0.0,
'w_quasi': 0.0,
'dt': dt,
'freqlist': freqlist,
'x': x,
'y': y,
'z': z,
'pdict': pdict
}
for freq in freqlist:
alph = np.sqrt(1j * 2 * np.pi * freq / (mu / rho))
uf, vf, wf = calcFourier(x, y, z, a, alph, freq, mu, c, U, tvec)
mydict['u_fourier'] += uf
mydict['v_fourier'] += vf
mydict['w_fourier'] += wf
uq, vq, wq = calcQuasiSteady_multfreqs(x, y, z, pdict, dt, tvec)
mydict['u_quasi'] = uq
mydict['v_quasi'] = vq
mydict['w_quasi'] = wq
pdict[
'Umsg'] = 'lambda t: np.array([U[0]*np.cos(2*np.pi*freq*t),0.0,0.0]) summed over frequency'
fname = os.path.join(basedir, basename)
F = open(fname + '.pickle', 'w')
Pickler(F).dump(mydict)
F.close()
def __setitem__(self, key, value):
with self._cache_write_lock:
self._cache[key] = value
f = StringIO()
p = Pickler(f, self._protocol)
p.dump(value)
self._storage.redis.hset(self._hash_key, key, f.getvalue())
def _zip_setitem(self, key, value):
"""``set-and-compress-item'' to dbase
"""
if self.writeback:
self.cache[key] = value
f = StringIO()
p = Pickler(f, self._protocol)
p.dump(value)
self.dict[key] = zlib.compress(f.getvalue(), self.compresslevel)
def deepcopy(obj):
"""Makes a deep copy of the object using the pickle mechanism.
"""
stream = StringIO()
p = Pickler(stream, 1)
p.dump(aq_base(obj))
stream.seek(0)
u = Unpickler(stream)
return u.load()
def pickle(obj, filename, binmode=0):
f = None
try:
f = open(filename, "wb")
p = Pickler(f, binmode)
p.dump(obj)
f.close()
f = None
finally:
if f: f.close()
def convert_and_save(self, data_obj):
pickle_file = open(self.file,'wb')
p = Pickler(pickle_file, -1)
p.dump(data_obj.lemmas)
p.dump(data_obj.rules)
p.dump(data_obj.gramtab)
p.dump(data_obj.prefixes)
p.dump(data_obj.possible_rule_prefixes)
p.dump(data_obj.endings)
if data_obj.rule_freq:
p.dump(data_obj.rule_freq)
def fast_encode():
# Only use in cases where you *know* the data contains only basic
# Python objects
pickler = Pickler(1)
pickler.fast = 1
dump = pickler.dump
def fast_encode(*args):
return dump(args, 1)
return fast_encode
def pickle(obj, filename, protocol=0):
f = None
try:
f = open(filename, "wb")
p = Pickler(f, protocol)
p.dump(obj)
f.close()
f = None
##print "Pickled", filename
finally:
if f: f.close()
def _zip_setitem(self, key, value):
""" ``set-and-compress-item'' to dbase
"""
ADD_ITEM = 'REPLACE INTO %s (key, value) VALUES (?,?)' % self.tablename
f = BytesIO()
p = Pickler(f, self.protocol)
p.dump(value)
blob = f.getvalue()
zblob = zlib.compress(blob, self.compression)
self.conn.execute(ADD_ITEM, (key, sqlite3.Binary(zblob)))
def send_command(self, *args):
verb_string = 'to %s %s ' % (self.role, args,)
# verbose('sendcommand args: '+`args`)
Pickler(self.input, 1).dump(args)
self.input.flush()
res = Unpickler(self.output).load()
verb_string = verb_string + '= %s' % (`res`,)
#print(verb_string) >> sys.stderr
if type(res) == types.StringType and res[:5] == 'Error':
raise TestRunException(res)
return res
def validate(self, validator_path, csl_schema_path, cslm_schema_path):
if self.opt.verbose:
print self.testname
m = re.match("(?sm).*version=\"1.1mlz1\".*", self.data["csl"])
if m:
rnc_path = cslm_schema_path
else:
rnc_path = csl_schema_path
tfd, tfilename = tempfile.mkstemp(dir=".")
os.write(tfd, self.data["csl"])
os.close(tfd)
jfh = os.popen("%s %s %s" % (validator_path, rnc_path, tfilename))
success = True
plural = ""
while 1:
line = jfh.readline()
if not line: break
line = line.strip()
e = re.match("^fatal:", line)
if e:
print line
sys.exit()
m = re.match(".*:([0-9]+):([0-9]+): *error:(.*)", line)
if m:
if success:
print "\n##"
print "#### Error%s in CSL for test: %s" % (plural,
self.hp)
print "##\n"
success = False
print " %s @ line %s" % (m.group(3).upper(), m.group(1))
plural = "s"
jfh.close()
os.unlink(tfilename)
if not success:
print ""
io = StringIO()
io.write(self.data["csl"])
io.seek(0)
linepos = 1
while 1:
cslline = io.readline()
if not cslline: break
cslline = cslline.rstrip()
print "%3d %s" % (linepos, cslline)
linepos += 1
pfh = open(self.pickle, "w+b")
pickler = Pickler(pfh)
pickler.dump((opt, self.pos))
sys.exit()
sys.stdout.write(".")
sys.stdout.flush()
def myExtension():
#dictionary of variables for both Stokes flow and viscoelastic flow
for N in [40]: #[20,40,80,160]:
pdict = dict(N=N,
M=N,
gridspc=0.0375,
origin=[-0.75, -0.75],
mu=1.0,
Wi=1.2)
pdict['beta'] = 1. / (2 * pdict['Wi'])
#set time parameters, save data every numskip time steps (default is every time step)
#note that I do not control the time step in the solver!!
# my time step only determines the maximum time step allowed
t0 = 0
totalTime = 30
dt = 5.e-2
#save file name
fname = os.path.expanduser('~/scratch/extension_noregrid')
# fname = os.path.expanduser('/scratch03/bcummins/mydata/ve/fourmill')
#first assign params
Np = 20
L = 0.78
h = L / (Np - 1)
pdict['eps'] = 2 * (0.75 * h)
pdict['U'] = 0.1
pdict['myVelocity'] = FourRollMill
#make the grid
M = pdict['M']
gridspc = pdict['gridspc']
l0 = mygrids.makeGridCenter(N, M, gridspc, pdict['origin'])
P0 = np.zeros((N, M, 2, 2))
P0[:, :, 0, 0] = 1.0
P0[:, :, 1, 1] = 1.0
y0 = np.append(l0.flatten(), P0.flatten())
#Viscoelastic run
print('Four roll mill flow, N = %02d' % N)
StateSave = mySolver(veExtensionUpdater, y0, t0, dt, totalTime, pdict,
1, 0)
#save the output
pdict['myVelocity'] = 'FourRollMill'
StateSave['pdict'] = pdict
StateSave['dt'] = dt
# print(StateSave.keys())
F = open(
fname + '_N%03d_Wi%02d_Time%02d.pickle' %
(N, int(round(pdict['Wi'])), int(round(totalTime))), 'w')
Pickler(F).dump(StateSave)
F.close()
def dump_pickle(self, archive):
logging.info("Pickling latest results to {0}.".format(self.pickle_path))
pickle_fp = self._get_pickle_file_pointer('wb')
pickler = Pickler(pickle_fp, protocol=2)
try:
pickler.dump(archive)
except PickleError as err:
logging.critical("Pickling failure. Error: {}".format(repr(err)))
finally:
pickle_fp.close()
self.pickle_exists = True
logging.info("Pickling complete to {0}".format(self.pickle_path))
return()
