def _init_unicode():
""" Prepare unicode property tables and key pattern. """
global _loaded
global _unicode_properties
global _unicode_key_pattern
if _use_cache is not None:
props = join(_use_cache, "%s_unicode_properties.cache" % _cache_prefix)
if (not exists(join(_use_cache, "%s_unicode_properties.cache" % _cache_prefix))):
_unicode_properties = _build_unicode_property_table((0x0000, 0x10FFFF))
_unicode_key_pattern = _build_unicode_key_pattern()
try:
with open(props, 'wb') as f:
pickle.dump(_unicode_key_pattern, f)
pickle.dump(_unicode_properties, f)
except Exception:
if exists(props):
unlink(props)
else:
try:
with open(props, 'rb') as f:
_unicode_key_pattern = pickle.load(f)
_unicode_properties = pickle.load(f)
except Exception:
if exists(props):
unlink(props)
_unicode_properties = _build_unicode_property_table((0x0000, 0x10FFFF))
_unicode_key_pattern = _build_unicode_key_pattern()
else:
_unicode_properties = _build_unicode_property_table((0x0000, 0x10FFFF))
_unicode_key_pattern = _build_unicode_key_pattern()
_loaded = True
def _init_unicode():
"""
Prepare unicode property tables and key pattern
"""
global _loaded
global _unicode_properties
global _unicode_key_pattern
if _use_cache is not None:
props = join(_use_cache, "%s_unicode_properties.cache" % _cache_prefix)
if (not exists(join(_use_cache, "%s_unicode_properties.cache" % _cache_prefix))):
_unicode_properties = _build_unicode_property_table((0x0000, 0x10FFFF))
_unicode_key_pattern = _build_unicode_key_pattern()
try:
with open(props, 'wb') as f:
pickle.dump(_unicode_key_pattern, f)
pickle.dump(_unicode_properties, f)
except:
if exists(props):
unlink(props)
else:
try:
with open(props, 'rb') as f:
_unicode_key_pattern = pickle.load(f)
_unicode_properties = pickle.load(f)
except:
if exists(props):
unlink(props)
_unicode_properties = _build_unicode_property_table((0x0000, 0x10FFFF))
_unicode_key_pattern = _build_unicode_key_pattern()
else:
_unicode_properties = _build_unicode_property_table((0x0000, 0x10FFFF))
_unicode_key_pattern = _build_unicode_key_pattern()
_loaded = True
def plot():
'''
PLOT PARAMETERS CAN BE ADJUSTED IN THIS METHOD
'''
# initialize plotter-instance with pickled stationbox:
stationBoxFile = open('stationBox.p','r')
pplotter = plotter(pickle.load(stationBoxFile))
cmt_lat = 37.10
cmt_lon = -3.69
# plot picks (Event coordinates needed to get relative distance):
CMTevent=model.Event(cmt_lat, cmt_lon, str_to_time("2010-04-11 22:08:15.500"),
"Spain_Durcal" , 625500 ,6.3)
pplotter.plotpicks(CMTevent)
# plot misfits:
try:
pickledResults = open('numpy_results.p','r')
resultArray = pickle.load(pickledResults)
finally:
pickledResults.close()
#pplotter.plotMisfits(data, _depths)
#.............................................................
# plot contour misfits for each model:
# 1st: load misfit array:
for phaseKey in ['p', 's', 'Scs', 'Scss']:
pplotter.plotContourMisfit(resultArray, _lats, _lons, _depths, phaseKey, _models[3])
#.............................................................
# set time shift file:
tShiftFiles = glob.glob('map/t_shift*.txt')
if tShiftFiles==[] or (tShiftFiles!=[] and (os.stat(tShiftFiles[0])[8] < os.stat('numpy_results.p')[8])):
for phase in ['p','s']:
for testmodel in _models:
print '(re)setting t-shift files for model: {0} and phase:{1}'.format(testmodel, phase)
depthfinder.setTshiftFile(CMTevent, np.searchsorted(_lats, CMTevent.lat, ),
np.searchsorted(_lons, CMTevent.lon), resultArray, testmodel, phase, _depths, maxdist )
else:
pass
for shiftmodel in _models:
pplotter.plotMisfitShiftMap('s',shiftmodel)
#.............................................................
#write horizontal layers to image files;
#pplotter.saveMisfitLayersAsImages(resultArray, _lats, _lons, _depths, 'models/premlocal1_Stich.nd','p', 630000, 600000 )
#asdf
CMTeventIndexLongitude = getIndexOfValueInArray(_lons, cmt_lon)
CMTeventIndexLatitude = getIndexOfValueInArray(_lats, cmt_lat)
pplotter.plotMisfitsForLocation(_models[0], CMTeventIndexLatitude, CMTeventIndexLongitude, resultArray, _depths)
#.............................................................
# create map
subprocess.call("map/durcal_stations.sh&", shell=True)
def from_iteration(cls, save_dir, i):
"""
Loads the data from a previous run
:param save_dir: directory for the data
:param i: index of iteration
:return:
"""
iteration_folder = save_dir + "/%03d" % (i, )
that = pickle.load(open(iteration_folder + "/bayesian_opt.pickle", "rb"))
if not isinstance(that, cls):
raise BayesianOptimizationError(BayesianOptimizationError.LOAD_ERROR, "not a robo instance")
new_x, X, Y, best_guess = pickle.load(open(iteration_folder + "/observations.pickle", "rb"))
return that, new_x, X, Y, best_guess
def create_index(number):
create_stop()
file = open('stop.pkl', 'rb')
stop = pickle.load(file)
file.close()
#inverse=defaultdict(lambda : defaultdict(int))
inverse = {}
col = {}
for i in range(0, 3):
doc = 'd' + str(i)
col[doc] = defaultdict(int)
for word in preprocess(i):
if word not in stop: # it uses the dictionary's hashing contrarly to using .key()
col[doc][word] += 1
if word in inverse.keys():
if doc not in inverse[word].keys():
inverse[word][doc] += 1
else:
inverse[word] = defaultdict(int)
inverse[word][doc] += 1
file = open('structure.pkl', 'wb')
pickle.dump(col, file)
file.close()
return col
def start_slices(self):
"""
Second step: start previously conflictive (non-standard) slices at FlowVisor.
"""
errors = []
slice_ids = []
# If 'conflictive_slice_ids' file exists, do the following.
# Otherwise warn and skip.
try:
f = open("%s/conflictive_slice_ids" % path, "r")
ids = pickle.load(f)
f.close()
os.remove("%s/conflictive_slice_ids" % path)
slices = filter_own_slices(ids)
for (counter, slice) in enumerate(slices):
aggs = slice.aggregates.filter(leaf_name="OpenFlowAggregate")
slice_id = str(settings.SITE_DOMAIN) + "_" + str(slice.id)
slice_ids.append({
"id": slice_id,
"keywords": ids[counter]['keywords']
})
for agg in aggs:
try:
print "Starting", slice_id, "at aggregate", str(agg)
time.sleep(FLOWVISOR_SLEEP_TIME)
with Timeout(TIMEOUT):
agg.as_leaf_class().client.proxy.create_slice(
slice_id, slice.project.name,
slice.project.description, slice.name,
slice.description,
slice.openflowsliceinfo.controller_url,
slice.owner.email,
slice.openflowsliceinfo.password,
agg.as_leaf_class()._get_slivers(slice))
# Starting slice at Expedient
slice.started = True
slice.save()
except Exception as e:
message = """Could not fix the naming in Aggregate Manager %s for slice with the following details: name:\t\t %s
FlowVisor name:\t\t %s
The cause of the error is: %s. Please try to fix it manually""" % (str(
agg.as_leaf_class()), slice.name, slice_id, e)
send_mail(
"OCF: error while standardizing Flowvisor slices",
message, "*****@*****.**",
[settings.ROOT_EMAIL])
errors.append(message)
# Save the slice IDs to grant flowspace nevertheless of other errors
f = open("%s/slice_ids_to_grant_fs" % path, "w")
pickle.dump(slice_ids, f)
f.close()
if errors:
return "\033[93mFailure while starting previously non-standard slices at FlowVisor: %s\033[0m" % str(
errors)
else:
return "\033[92mSuccessfully started previously non-standard slices at FlowVisor\033[0m"
except Exception as e:
print e
return "\033[93mCould not access file with slice IDs. Skipping...\033[0m\n"
def random_h(G, stringie, dimension, num_iter, max_index):
h = []
ht = []
for i in range(num_iter):
namefile = stringie + str(i) + '.pck'
gen_temp = pickle.load(open(namefile))
if gen_temp.has_key(dimension):
h.append(network_hollowness(gen_temp[dimension], max_index))
ht.append(
network_weighted_hollowness(G, gen_temp[dimension], max_index))
print 'h=' + str(np.mean(h)) + '+-' + str(np.std(h))
print 'ht=' + str(np.mean(ht)) + '+-' + str(np.std(ht))
if len(h) >= 1:
a = np.mean(h)
b = np.std(h)
else:
a = 0
b = 0
if len(ht) > 1:
c = np.mean(ht)
d = np.std(ht)
else:
c = 0
d = 0
return a, b, c, d
def test_pickle(obj):
"""Test if an object can successfully be pickled and loaded again
Returns True if succeeds
False if fails
"""
import pickle
# sys.setrecursionlimit(10000)
path = 'test.p.tmp'
if os.path.isfile(path):
os.remove(path) # remove temp file
try:
with open(path, 'wb') as f:
pickle.dump(obj, f)
print('Pickled object')
with open(path, 'rb') as f:
out = pickle.load(f)
print('Loaded object')
except Exception as e:
print('{}'.format(e))
return False
if os.path.isfile(path):
print('Pickled file size: {:g} Bytes'.format(os.path.getsize(path)))
os.remove(path) # remove temp file
import pdb; pdb.set_trace()
print('In:\n{}\nOut:\n{}'.format(out, obj))
if not isinstance(obj, dict):
out = out.__dict__
obj = obj.__dict__
if compare_dict(out, obj):
return True
else:
return False
def from_iteration(cls, save_dir, i):
"""
Loads the data from a previous run
:param save_dir: directory for the data
:param i: index of iteration
:return:
"""
iteration_folder = save_dir + "/%03d" % (i, )
that = pickle.load(
open(iteration_folder + "/bayesian_opt.pickle", "rb"))
if not isinstance(that, cls):
raise BayesianOptimizationError(
BayesianOptimizationError.LOAD_ERROR, "not a robo instance")
new_x, X, Y, best_guess = pickle.load(
open(iteration_folder + "/observations.pickle", "rb"))
return that, new_x, X, Y, best_guess
def load(self, file_name):
"""Load the buffer from a file.
@param file_name
"""
self.buffer = pickle.load(open(file_name, 'rb'))
self.size = len(self.buffer)
def _load_data(datafile, datafilefmt):
'''
Load the data file from the last run, or if it does not exist, return an
empty dictionary. Backup any existing data in the process.
'''
if not os.path.isfile(datafile):
data = {}
else:
if datafilefmt == 'json':
with open(datafile, encoding='utf8') as f:
data = json.load(f)
elif datafilefmt == 'json.zip':
with zipfile.ZipFile(datafile) as z:
fname = os.path.split(datafile)[1].rsplit('.', 1)[0]
with z.open(fname) as f:
data = json.loads(f.read().decode('utf8'))
elif datafilefmt in ('pickle', 'pkl'):
with open(datafile, 'rb') as f:
data = pickle.load(f)
else:
raise ValueError(
'Invalid data file format {0}'.format(datafilefmt))
# Create 2 levels of backups--maybe a little paranoid, but safer
# The data file is only written after everything is complete and there
# are no errors, so copying it in this manner doesn't risk overwriting
# valid backups with bad data
if os.path.isfile(datafile + '.backup'):
shutil.copy(datafile + '.backup', datafile + '.backup2')
shutil.copy(datafile, datafile + '.backup')
return data
def get_ngrams(N=3):
'''Return doc- and corpus-level ngram counts for each word type.
Counts are based on n-grams of size 1 through N (inclusive).
Ngrams are stored and returned as a dictionary:
ngrams = {
'doc': {
doc1.id : {
0: {
ngram1: <count>,
ngram2: <count>,
...
},
...
},
...
},
'corpus': {
0: {
ngram1: <count>,
ngram2: <count>,
...
},
...
},
}
'''
try:
with open(NGRAM_FILE % N, 'rb') as f:
return pickle.load(f)
except (IOError, EOFError):
return _get_ngrams(N)
def getDates():
try:
import cpickle as pickle
except ImportError:
import pickle as pickle
import datetime
dateList=[]
dateHistFile = "dateList.d"
try:
dateList=pickle.load( open( dateHistFile, "rb" ) )
except IOError:
#Change this to the min date instead of a fixed date
#But then we need to consider if the holiday dates are in the same range
dateList = map(okDate,[ datetime.datetime(2012, 12, 31, 0, 0) + datetime.timedelta(days=x) for x in range(0,365)])
dateList = [dateVal for dateVal in dateList if dateVal is not None]
#If this file is being run or loaded, then we need to recreate the pickled dateList
pickle.dump( dateList, open( dateHistFile, "wb" ))
return dateList
def _load_data(datafile, datafilefmt):
'''
Load the data file from the last run, or if it does not exist, return an
empty dictionary. Backup any existing data in the process.
'''
if not os.path.isfile(datafile):
data = {}
else:
if datafilefmt == 'json':
with open(datafile, encoding='utf8') as f:
data = json.load(f)
elif datafilefmt == 'json.zip':
with zipfile.ZipFile(datafile) as z:
fname = os.path.split(datafile)[1].rsplit('.', 1)[0]
with z.open(fname) as f:
data = json.loads(f.read().decode('utf8'))
elif datafilefmt in ('pickle', 'pkl'):
with open(datafile, 'rb') as f:
data = pickle.load(f)
else:
raise ValueError('Invalid data file format {0}'.format(datafilefmt))
# Create 2 levels of backups--maybe a little paranoid, but safer
# The data file is only written after everything is complete and there
# are no errors, so copying it in this manner doesn't risk overwriting
# valid backups with bad data
if os.path.isfile(datafile + '.backup'):
shutil.copy(datafile + '.backup', datafile + '.backup2')
shutil.copy(datafile, datafile + '.backup')
return data
def getDates():
try:
import cpickle as pickle
except ImportError:
import pickle as pickle
import datetime
dateList = []
dateHistFile = "dateList.d"
try:
dateList = pickle.load(open(dateHistFile, "rb"))
except IOError:
#Change this to the min date instead of a fixed date
#But then we need to consider if the holiday dates are in the same range
dateList = map(okDate, [
datetime.datetime(2012, 12, 31, 0, 0) + datetime.timedelta(days=x)
for x in range(0, 365)
])
dateList = [dateVal for dateVal in dateList if dateVal is not None]
#If this file is being run or loaded, then we need to recreate the pickled dateList
pickle.dump(dateList, open(dateHistFile, "wb"))
return dateList
def load_luma_map(luma_file=LUMA_FILE):
print "LOADING LUMA MAP", luma_file
try:
with open(luma_file) as f:
global LUMA_MAP
LUMA_MAP = pickle.load(f)
print_luma_completion()
except Exception, e:
print "WARNING: NOT LOADING LUMA MAP", e
def load_luma_observations():
global LUMA_OBS
if os.path.exists(CHANGES_FILE):
try:
with open(CHANGES_FILE) as f:
LUMA_OBS = pickle.load(f)
print "LOADED %s LUMA OBSERVATIONS" % len(LUMA_OBS)
except Exception, e:
print "EXCEPT", e
def init_last_iteration(self):
"""
Loads the last iteration from a previously stored run
:return: the previous observations
"""
max_iteration = self._get_last_iteration_number()
iteration_folder = os.path.join(self.save_dir, "%03d" % (max_iteration, ))
that = pickle.load(open(os.path.join(iteration_folder, "bayesian_opt.pickle"), "rb"))
self.objective_fkt = that.objective_fkt
self.acquisition_fkt = that.acquisition_fkt
self.model = that.model
self.maximize_fkt = that.maximize_fkt
self.X_lower = that.X_lower
self.X_upper = that.X_upper
self.dims = that.dims
return pickle.load(open(iteration_folder + "/observations.pickle", "rb"))
def loadPickle(_fileName=PickleFileG):
'''读取上一次执行结果，方便与最新数据进行比较'''
if os.path.exists(_fileName):
f=file(_fileName,'rb')
_Data = pickle.load(f)
f.close()
return _Data
else:
return {'date':'','data':{}}
def load_luma_map(luma_file=LUMA_FILE):
print "LOADING LUMA MAP", luma_file
try:
with open(luma_file) as f:
global LUMA_MAP
LUMA_MAP = pickle.load(f)
print_luma_completion()
except Exception, e:
print "WARNING: NOT LOADING LUMA MAP", e
def configure(self, options, conf):
self.hash_file = join(conf.workingDir, self.hash_file)
if isfile(self.hash_file):
log.debug("Loading last known hashes and dependency graph")
with open(self.hash_file, 'r') as f:
data = load(f)
self._known_hashes = data['hashes']
self._known_graph = data['graph']
Plugin.configure(self, options, conf)
def load_luma_observations():
global LUMA_OBS
if os.path.exists(CHANGES_FILE):
try:
with open(CHANGES_FILE) as f:
LUMA_OBS = pickle.load(f)
print "LOADED %s LUMA OBSERVATIONS" % len(LUMA_OBS)
except Exception, e:
print "EXCEPT", e
def loadData():
Tkinter.Tk().withdraw()
fp = tkFileDialog.askopenfilename()
print("Opening " + fp + "...")
with open(fp, 'r') as file:
emgs = pickle.load(file)
print "Loaded data file."
return emgs
def save_temps(filename):
f = open(filename, 'w')
data = cpickle.load(f)
for i in range(1,9):
temp = read_temp(i)
now = datetime.datetime.now()
data[now] = [i, temp]
time.sleep(0.5)
cpickle.dump(data, f)
f.close()
def configure(self, options, conf):
self.hash_file = os.path.join(conf.workingDir, self.hash_file)
if os.path.isfile(self.hash_file):
log.debug("Loading last known hashes and dependency graph")
f = open(self.hash_file, 'r')
data = load(f)
f.close()
self._known_hashes = data['hashes']
self._known_graph = data['graph']
Plugin.configure(self, options, conf)
def init_last_iteration(self):
"""
Loads the last iteration from a previously stored run
:return: the previous observations
"""
max_iteration = self._get_last_iteration_number()
iteration_folder = os.path.join(self.save_dir,
"%03d" % (max_iteration, ))
that = pickle.load(
open(os.path.join(iteration_folder, "bayesian_opt.pickle"), "rb"))
self.objective_fkt = that.objective_fkt
self.acquisition_fkt = that.acquisition_fkt
self.model = that.model
self.maximize_fkt = that.maximize_fkt
self.X_lower = that.X_lower
self.X_upper = that.X_upper
self.dims = that.dims
return pickle.load(
open(iteration_folder + "/observations.pickle", "rb"))
def __init__(self):
DIR = dirname(__file__)
morfessor_file = join(DIR, 'data/finnsyll-morfessor.bin')
ngram_file = join(DIR, 'data/finnsyll-ngrams.pickle')
io = morfessor.MorfessorIO()
self.model = io.read_binary_model_file(morfessor_file)
self.constraints = CONSTRAINTS
self.constraint_count = len(CONSTRAINTS)
with open(ngram_file, 'rb') as f:
self.ngrams, self.vocab, self.total = pickle.load(f)
def getObj(dataFile=None):
try:
import cpickle as pickle
except ImportError:
import pickle as pickle
dataObj={}
try:
dataObj=pickle.load( open( dataFile, "rb" ) )
except IOError:
return dataObj
return dataObj
def pickle_load(path, base=None, **kwargs):
if isinstance(path, Path):
path = str(path)
if base is not None:
path = os.path.join(base, path)
if isinstance(path, basestring):
if path[-2:] != '.p':
path += '.p'
with open(path, 'rb') as f:
try:
out = pickle.load(f, **kwargs)
except EOFError as e:
logger.error('path "{}" is not a pickle file. {}'.format(path, e))
raise e
elif isinstance(path, file):
out = pickle.load(path, **kwargs)
path.close()
else:
raise ValueError('Unexpected path format')
return out
def run_job(f, *args, **kwargs):
''' Internal function, runs the function in a remote process.
Uses fork() to perform it.
Availability: Unix
'''
out_file = tempfile.mkstemp()
os.close(out_file[0])
pid = os.fork()
if pid != 0:
# parent: wait for the child
pid, status = os.waitpid(pid, 0)
# read output file
#print('read from', os.getpid(), ':', out_file[1])
if os.stat(out_file[1]).st_size == 0:
# child did not write anything
os.unlink(out_file[1])
raise OSError('child did not output anything')
if status != 0:
os.unlink(out_file[1])
raise RuntimeError('subprocess error: %d' % status)
result = pickle.load(open(out_file[1], 'rb'))
os.unlink(out_file[1])
# traceback objects cannot be pickled...
#if isinstance(result, tuple) and len(result) == 3 \
#and isinstance(result[1], Exception):
## result is an axception with call stack: reraise it
#raise result[0], result[1], result[2]
if isinstance(result, Exception):
raise result
return result
# child process
try:
try:
#print('exec in', os.getpid(), ':', f, args, kwargs)
result = f(*args, **kwargs)
#print('OK')
except Exception as e:
# traceback objects cannot be pickled...
#result = (type(e), e, sys.exc_info()[2])
result = e
#print('write:', out_file[1], ':', result)
try:
pickle.dump(result, open(out_file[1], 'wb'), protocol=2)
except Exception as e:
print('pickle failed:', e, '\nfor object:', type(result))
finally:
# sys.exit() is not enough
os._exit(0)
def readFromFile(self, filename):
""" Read contacts in from a file. """
try:
f = open(filename, "rb")
addresses = pickle.load(f)
except IOError:
QMessageBox.information(self, "Unable to open file: %s" % filename)
finally:
f.close()
if len(addresses) == 0:
QMessageBox.information(self, "No contacts in file: %s" % filename)
else:
for address in addresses:
self.addEntry(address["name"], address["address"])
def readSession(filepath, helper=None):
"""
Loads a dictionary session from file.
:param filepath: path to load session file.
:param helper: function to pos-process session file
:return: session
"""
# safely read session file
with secure_open(filepath, 'rb') as logger:
session = serializer.load(logger) # get session
if helper:
return helper(session)
else:
return session
def readFromFile(self, filename):
""" Read contacts in from a file. """
try:
f = open(filename, "rb")
addresses = pickle.load(f)
except IOError:
QMessageBox.information(self, "Unable to open file: %s" % filename)
finally:
f.close()
if len(addresses) == 0:
QMessageBox.information(self, "No contacts in file: %s" % filename)
else:
for address in addresses:
self.addEntry(address["name"], address["address"])
def load(cls, filename):
""" Create an engine given metadata from a pickle file.
Parameters
----------
filename : str
"""
with open(filename, 'rb') as f:
dat = pkl.load(f)
self = cls(**dat['init_args'])
for key, val in dat['cls_attrs'].items():
setattr(self, '_' + key, val)
random.setstate(dat['rng_state']['py'])
np.random.set_state(dat['rng_state']['np'])
return self
def load(cls, filename):
""" Create an engine given metadata from a pickle file.
Parameters
----------
filename : str
"""
with open(filename, 'rb') as f:
dat = pkl.load(f)
self = cls(**dat['init_args'])
for key, val in dat['cls_attrs'].items():
setattr(self, '_' + key, val)
random.setstate(dat['rng_state']['py'])
np.random.set_state(dat['rng_state']['np'])
return self
def start_slices(self):
"""
Second step: start previously conflictive (non-standard) slices at FlowVisor.
"""
errors = []
slice_ids = []
# If 'conflictive_slice_ids' file exists, do the following.
# Otherwise warn and skip.
try:
f = open("%s/conflictive_slice_ids" % path,"r")
ids = pickle.load(f)
f.close()
os.remove("%s/conflictive_slice_ids" % path)
slices = filter_own_slices(ids)
for (counter, slice) in enumerate(slices):
aggs = slice.aggregates.filter(leaf_name="OpenFlowAggregate")
slice_id = str(settings.SITE_DOMAIN) + "_" + str(slice.id)
slice_ids.append({"id":slice_id, "keywords":ids[counter]['keywords']})
for agg in aggs:
try:
print "Starting", slice_id, "at aggregate", str(agg)
time.sleep(FLOWVISOR_SLEEP_TIME)
with Timeout(TIMEOUT):
agg.as_leaf_class().client.proxy.create_slice(slice_id, slice.project.name,slice.project.description,slice.name, slice.description, slice.openflowsliceinfo.controller_url, slice.owner.email, slice.openflowsliceinfo.password, agg.as_leaf_class()._get_slivers(slice))
# Starting slice at Expedient
slice.started = True
slice.save()
except Exception as e:
message = """Could not fix the naming in Aggregate Manager %s for slice with the following details: name:\t\t %s
FlowVisor name:\t\t %s
The cause of the error is: %s. Please try to fix it manually""" % (str(agg.as_leaf_class()),slice.name, slice_id, e)
send_mail("OCF: error while standardizing Flowvisor slices", message, "*****@*****.**", [settings.ROOT_EMAIL])
errors.append(message)
# Save the slice IDs to grant flowspace nevertheless of other errors
f = open("%s/slice_ids_to_grant_fs" % path,"w")
pickle.dump(slice_ids, f)
f.close()
if errors:
return "\033[93mFailure while starting previously non-standard slices at FlowVisor: %s\033[0m" % str(errors)
else:
return "\033[92mSuccessfully started previously non-standard slices at FlowVisor\033[0m"
except Exception as e:
print e
return "\033[93mCould not access file with slice IDs. Skipping...\033[0m\n"
def getHistory(): #add the
try:
import cpickle as pickle
except ImportError:
import pickle as pickle
import datetime
import time
historyFile = "stockHist.d"
try:
histDict = pickle.load(open(historyFile, "rb"))
# Need to check if the data files have changed and if so recreate
# Check the max date in the database against the retrieved array
except IOError:
from sys import getsizeof
from symbols import symbolList
from processFuncs import writeLog
histDict = {}
recHistory = []
dateSearch = datetime.datetime(
2013, 1, 2, 0, 0
) #We know this is the starting date. Should probably use the tradingDates.py
db = openData('stockData')
useIndex(db, 'nasdaq', 'Date', 'Symbol')
#db.nasdaq.ensure_index([('Date','Symbol')])
writeLog("Creating Price History")
#Symbol,Date,Open,High,Low,Close,Volume
#prices={'AAPL':[('20130201',433.45,434.30,432.33,436.3,2030400),('20130202',433.55,434.50,432.33,436.3,2030400)]}
#.strftime("%Y%m%d")
for symbol in symbolList:
recHistory = map(lambda x: (x['Date'].combine(x['Date'].date(),datetime.time(0,0,0)),x['Open'],x['High'],x['Low'],x['Close']),\
db.nasdaq.find({'Symbol':symbol},{'Date':1,'Open': 1,'High':1,'Low':1,'Close': 1, '_id':0 }).sort('Date',1))
#print recHistory
#raw_input()
histDict[symbol] = recHistory
recHistory = []
pickle.dump(histDict, open(historyFile, "wb"))
return histDict
def getHistory(): #add the
try:
import cpickle as pickle
except ImportError:
import pickle as pickle
import datetime
import time
historyFile = "stockHist.d"
try:
histDict=pickle.load( open( historyFile, "rb" ) )
# Need to check if the data files have changed and if so recreate
# Check the max date in the database against the retrieved array
except IOError:
from sys import getsizeof
from symbols import symbolList
from processFuncs import writeLog
histDict={}
recHistory=[]
dateSearch = datetime.datetime(2013, 1, 2, 0, 0) #We know this is the starting date. Should probably use the tradingDates.py
db=openData('stockData')
useIndex(db,'nasdaq','Date','Symbol')
#db.nasdaq.ensure_index([('Date','Symbol')])
writeLog("Creating Price History")
#Symbol,Date,Open,High,Low,Close,Volume
#prices={'AAPL':[('20130201',433.45,434.30,432.33,436.3,2030400),('20130202',433.55,434.50,432.33,436.3,2030400)]}
#.strftime("%Y%m%d")
for symbol in symbolList:
recHistory = map(lambda x: (x['Date'].combine(x['Date'].date(),datetime.time(0,0,0)),x['Open'],x['High'],x['Low'],x['Close']),\
db.nasdaq.find({'Symbol':symbol},{'Date':1,'Open': 1,'High':1,'Low':1,'Close': 1, '_id':0 }).sort('Date',1))
#print recHistory
#raw_input()
histDict[symbol]=recHistory
recHistory=[]
pickle.dump( histDict, open( historyFile, "wb" ))
return histDict
def intensityMap(self, plot=False):
path = self._gfile.split("/")[-1]
nR = 100
nphi = 200
try:
fieldlines = pickle.load(open("fieldline_store/" + path + "__fl.p", "r"))
except:
fieldlines = []
Rstart = 1.37
phistart = 160.0
dR = 0.001
dphi = 0.2
for i in np.arange(nR):
R = Rstart + i * dR
fline = self.tracer.trace(R, 0.0, phistart * 2.0 * np.pi / 360.0, mxstep=10000, ds=0.01)
for j in np.arange(nphi):
fline.rotate_toroidal(dphi * 2.0 * np.pi / 360.0)
line = self.projectFieldline(fline)
indsR = np.where(np.array(fline.R) > 0.6)[0]
inds = np.where(np.abs(fline.Z)[indsR] < 1.0)[0]
fieldlines.append(line[inds])
pickle.dump(fieldlines, open("fieldline_store/" + path + "__fl.p", "w"))
frame = self.bgsub.apply(self._currentframeData)
intensity = []
for i in np.arange(len(fieldlines)):
line = fieldlines[i]
temp = []
points = np.zeros(frame.shape)
for j in np.arange(line.shape[0]):
yind, xind = int(line[j, 0]), int(line[j, 1])
if xind > 0 and xind < frame.shape[0] and yind > 0 and yind < frame.shape[1]:
points[xind, yind] = 1.0
intensity.append(np.sum(points * frame))
intensity = np.array(intensity).reshape((nR, nphi))
if plot:
fig = plt.figure()
levels = np.linspace(np.min(intensity), np.max(intensity), 200)
plt.contourf(np.linspace(1.37, 1.47, 100), np.linspace(160, 200, 200), intensity.T, levels=levels)
plt.show()
else:
return intensity
def new_f(*args, **kwargs):
compressed = ''
if len(args) > 0:
compressed = '_' + '_'.join([str(arg)[:10] for arg in args])
if len(kwargs) > 0:
compressed += '_' + '_'.join([(str(k)+str(v))[:10] for k,v in kwargs])
filename = '%s%s.pickle' % (f.__name__, compressed)
if os.path.exists(filename):
pickled = open(filename, 'rb')
result = pickle.load(pickled)
pickled.close()
else:
result = f(*args, **kwargs)
pickled = open(filename, 'wb')
pickle.dump(result, pickled)
pickled.close()
return result
def new_f(*args, **kwargs):
compressed = ''
if len(args) > 0:
compressed = '_' + '_'.join([str(arg)[:10] for arg in args])
if len(kwargs) > 0:
compressed += '_' + '_'.join([(str(k)+str(v))[:10] for k,v in kwargs])
filename = '%s%s.pickle' % (f.__name__, compressed)
if os.path.exists(filename):
pickled = open(filename, 'rb')
result = pickle.load(pickled)
pickled.close()
else:
result = f(*args, **kwargs)
pickled = open(filename, 'wb')
pickle.dump(result, pickled)
pickled.close()
return result
def new_f(*args, **kwargs):
compressed = ''
if len(args) > 0:
compressed = '_' + '_'.join([str(arg)[:10] for arg in args])
if len(kwargs) > 0:
compressed += '_' + '_'.join([(str(k) + str(v))[:10] for k, v in kwargs])
logging.debug("Input File Name " +args[1])
filename = '%s.pickle' % (args[1])
logging.debug("Pickling file name %s "% filename)
if os.path.exists(filename):
pickled = open(filename, 'rb')
result = pickle.load(pickled)
pickled.close()
else:
result = f(*args, **kwargs)
pickled = open(filename, 'wb')
pickle.dump(result, pickled)
pickled.close()
return result
def get_posteriors(N=3):
'''Return doc- and corpus-level posterior probabilities of each word type.
Posteriors are based on n-grams of size 1 through N (inclusive).
Posteriors are stored and returned as a dictionary:
posteriors = {
'doc': {
doc1.id : {
0: {
tok1.id: <posterior>,
tok2.id: <posterior>,
...
},
...
},
...
},
'corpus': {
0: {
tok1.id: <posterior>,
tok2.id: <posterior>,
...
},
...
},
}
'''
try:
with open(POSTERIOR_FILE % N, 'rb') as f1:
with open(INFORMATIVITY_FILE % N, 'rb') as f2:
assert f2
return pickle.load(f1)
except (IOError, EOFError, AssertionError):
return _get_posteriors(N)
def __init__(self, **karg):
super(Cache2, self).__init__()
Cache2._CACHES.append(self)
_karg = {'poolsize': 50, 'base': None}
_karg.update(karg)
self.poolsize = _karg['poolsize']
self.pool = dict()
self.ttl = dict()
self.tick = 0
self.base = _karg['base']
if _karg['base']:
if _karg['base'] not in Cache2._CACHEFILES:
Cache2._CACHEFILES.add(_karg['base'])
else:
raise Exception, 'Basefiles conflicts [{0}]'.\
format(_karg['base'])
if os.path.exists(_karg['base']):
with open(_karg['base'],'rb') as basefile:
self.poolsize, self.pool, self.ttl, self.tick = pickle.load(
basefile)
self.base = _karg['base']
def __init__(self, **karg):
super(Cache2, self).__init__()
Cache2._CACHES.append(self)
_karg = {'poolsize': 50, 'base': None}
_karg.update(karg)
self.poolsize = _karg['poolsize']
self.pool = dict()
self.ttl = dict()
self.tick = 0
self.base = _karg['base']
if _karg['base']:
if _karg['base'] not in Cache2._CACHEFILES:
Cache2._CACHEFILES.add(_karg['base'])
else:
raise Exception, 'Basefiles conflicts [{0}]'.\
format(_karg['base'])
if os.path.exists(_karg['base']):
with open(_karg['base'], 'rb') as basefile:
self.poolsize, self.pool, self.ttl, self.tick = pickle.load(
basefile)
self.base = _karg['base']
def get_informativity(N=3):
'''Return doc- and corpus-level informativity of each word type.
Informativity is based on n-grams of size 1 through N (inclusive).
Informativity results are stored and returned as a dictionary:
informativity = {
'doc': {
doc1.id : {
sigma1: {
1: <bigram_informativity>,
2: <trigram_informativity>,
...
},
...
},
...
},
'corpus': {
sigma1: {
1: <bigram_informativity>,
2: <trigram_informativity>,
...
},
...
},
}
'''
try:
with open(INFORMATIVITY_FILE % N, 'rb') as f:
return pickle.load(f)
except (IOError, EOFError):
# informativity is calculated at the same time as posteriors
get_posteriors(N)
return get_informativity(N)
def random_h(G,stringie,dimension,num_iter,max_index): h=[]; ht=[]; for i in range(num_iter): namefile=stringie+str(i)+'.pck'; gen_temp=pickle.load(open(namefile)); if gen_temp.has_key(dimension): h.append(network_hollowness(gen_temp[dimension],max_index)); ht.append(network_weighted_hollowness(G,gen_temp[dimension],max_index)); print 'h='+str(np.mean(h))+'+-'+str(np.std(h)); print 'ht='+str(np.mean(ht))+'+-'+str(np.std(ht)); if len(h)>=1: a=np.mean(h); b=np.std(h); else: a=0; b=0; if len(ht)>1: c=np.mean(ht); d=np.std(ht); else: c=0; d=0; return a,b,c,d;
def load_pyeq(equilibrium,filename):
try:
import cpickle as pickle
except:
import pickle
#First try loading filename as given
try:
file = open(filename,'rb')
except:
#Next try append the .pyeq filename
filename += '.pyeq'
try:
file = open(filename,'rb')
except:
raise IOError('Error: unable to open file --> '+filename)
try:
eq = pickle.load(file)
except:
raise IOError('Error: unable to unpickle data from file --> '+filename)
return eq
def test_variant_ordering_no_stress(self):
# ensure that the syllabifier returns variants in order from most
# preferred to least preferred
F = FinnSyll(split=True, variation=True, rules=False, stress=False)
with open('tests/ranked_sylls.pickle', 'rb') as f:
pairs = pickle.load(f)
errors = 0
for i, expected in pairs.items():
try:
test = F.syllabify(unicode(i, 'utf-8').lower())
except (TypeError, NameError):
test = F.syllabify(i.lower())
try:
self.assertEqual(test, expected)
except AssertionError as e:
errors += 1
message = ''
for line in e.message.split('\n'):
if line.startswith('-'):
message += line + '\n'
elif line.startswith('+'):
message += line
print(message + '\n')
if errors:
raise AssertionError(errors)
def getRatios():
try:
import cpickle as pickle
except ImportError:
import pickle as pickle
import datetime
pairsFile = "ratioHistory.d"
try:
pairsDict = pickle.load(open(pairsFile, "rb"))
except IOError:
from sys import getsizeof
from symbols import symbolList
import numpy as np
from processFuncs import writeLog
from math import log
#Then we need to create the file
# Structure {Symbol:[12.12,13,13.2...],...]
# In our program, we need to retrieve the history array of open,close values and then append the latest values for each
# symbol
# For testing, just retrieve the latest history
from createHistory import getHistory
# This is how we get the price ratio
# This will get us the average price and then divide them
# np.divide(np.average(histDict['AAPL']),np.average(histDict['AMZN']))
# This will create an array that shows the history of the price ratios
# np.divide(histDict['AAPL'],histDict['AMZN'])
# Find the standard deviation of the price ratios
# np.std(np.divide(histDict['AAPL'],histDict['AMZN']))
pairsDict = {}
pairsHistory = []
symbol1Arry = []
symbol2Arry = []
savePair = 0
dupeList = symbolList[:]
stockHist = getHistory(
) #This will be the entire history in a dictionary with key=symbol the dates are based on order
writeLog("Creating Pairs")
for symbol in symbolList:
# Don't need to do this since we already have a history function
#symbolArry = stockHist[symbol]#map(lambda x: x['Close'],db.nasdaq.find({'Symbol':symbol},{'Close': 1, '_id':0 }))
symbolArry = map(lambda x: (x[0], x[4]), stockHist[symbol])
for symbol2 in dupeList:
symbol2Arry = []
if symbol2 == symbol:
savePair = 0
continue
#symbol2Arry = stockHist[symbol2] #map(lambda x: x['Close'],db.nasdaq.find({'Symbol':symbol2},{'Close': 1, '_id':0 }))
symbol2Arry = map(lambda x: (x[0], x[4]), stockHist[symbol2])
minLen = min(len(symbolArry), len(symbol2Arry))
if minLen > 0:
symbolArry = symbolArry[-minLen:]
symbol2Arry = symbol2Arry[-minLen:]
savePair = 1
#if len(symbolArry)==len(symbol2Arry):
# savePair=1
if savePair:
# Need to create another dictionary that stores the average and standard deviation
# Also, need to keep track of correlation
# For each day we need to divide
symbol2dict = dict(symbol2Arry)
doPairAppend = pairsHistory.append
for day in symbolArry:
dayDate = day[0]
dayClose = day[1]
if dayDate not in symbol2dict:
continue
day2Close = symbol2dict[dayDate]
doPairAppend(
(dayDate,
log(
round(
np.divide(float(dayClose),
float(day2Close)), 5))))
#print pairsHistory
#print symbol,symbol2
#raw_input()
pairsDict[("%s %s") % (symbol, symbol2)] = pairsHistory
if False:
for day in symbolArry:
dayDate = day[0]
day2Close = day[1]
#The date is stored in GMT time, so sometimes it has standard time and sometimes it's daylight
#time. Have to remove the GMT portion. Not sure if in import or when creating history.
if dayDate not in symbol2dict:
continue
dayClose = symbol2dict[dayDate]
pairsHistory.append(
(dayDate,
round(np.divide(day2Close, dayClose), 5)))
pairsDict[("%s %s") % (symbol2, symbol)] = pairsHistory
#pairsHistory=np.divide(symbol2Arry,symbolArry,)
#pairsDict[("%s %s")%(symbol2,symbol)]=pairsHistory
pairsHistory = []
symbol1Arry = []
symbol2Arry = []
savePair = 0
#if len(dupeList):
# dupeList.pop(0)
#else:
# break
pickle.dump(pairsDict, open(pairsFile, "wb"))
return pairsDict
updates=updates)
def train(self, datas):
total_cost = 0.0
random.shuffle(datas)
for i in range(len(datas)):
#for i in range(1):
batch_sentences = datas[i][0]
batch_answers = datas[i][1]
cost, pred = self.train_model(batch_sentences, batch_answers, 0)
#print str(pred) +"\t"+ str(batch_answers)+"\t"+str(norm_weight)
total_cost += cost
print "total cost is " + str(total_cost / len(datas))
data_set = pickle.load(open("data/imdb.pkl"))
"""
train data : 20000
test data : 5000
train_set[0] : uni-gram word sequence
train_set[1] : class of data
test_set[0] : uni-fram word sequence
test_set[1] : class of data
전체 단어 집합 : 91189
훈련 데이터 입력 단어 :
"""
train_set = data_set['train']
test_set = data_set['test']
def load(self, fileobject=None):
fileobject, close = file_open(fileobject or self._cachefile, 'rb')
self._symcache = pickle.load(fileobject)
if close:
fileobject.close()
def __init__(self, name):
self.name = name
self.name_backwards = name[::-1]
return
data = []
data.append(SimpleObject('pickle'))
data.append(SimpleObject('cpickle'))
data.append(SimpleObject('last'))
#Simulate a file with StringIO
out_s = StringIO()
#Write to the stream
for o in data:
print 'WRITING : %s (%s)' %(o.name, o.name_backwards)
pickle.dump(o, out_s)
out_s.flush()
#Set up a read-able stream
in_s = StringIO(out_s.getvalue())
#Read the data
while True:
try:
o = pickle.load(in_s)
except EOFError:
break
else:
print 'READ : %s (%s)' % (o.name, o.name_backwards)
def getRatios():
try:
import cpickle as pickle
except ImportError:
import pickle as pickle
import datetime
pairsFile = "ratioHistory.d"
try:
pairsDict=pickle.load( open( pairsFile, "rb" ) )
except IOError:
from sys import getsizeof
from symbols import symbolList
import numpy as np
from processFuncs import writeLog
from math import log
#Then we need to create the file
# Structure {Symbol:[12.12,13,13.2...],...]
# In our program, we need to retrieve the history array of open,close values and then append the latest values for each
# symbol
# For testing, just retrieve the latest history
from createHistory import getHistory
# This is how we get the price ratio
# This will get us the average price and then divide them
# np.divide(np.average(histDict['AAPL']),np.average(histDict['AMZN']))
# This will create an array that shows the history of the price ratios
# np.divide(histDict['AAPL'],histDict['AMZN'])
# Find the standard deviation of the price ratios
# np.std(np.divide(histDict['AAPL'],histDict['AMZN']))
pairsDict={}
pairsHistory=[]
symbol1Arry=[]
symbol2Arry=[]
savePair=0
dupeList=symbolList[:]
stockHist = getHistory() #This will be the entire history in a dictionary with key=symbol the dates are based on order
writeLog("Creating Pairs")
for symbol in symbolList:
# Don't need to do this since we already have a history function
#symbolArry = stockHist[symbol]#map(lambda x: x['Close'],db.nasdaq.find({'Symbol':symbol},{'Close': 1, '_id':0 }))
symbolArry = map(lambda x:(x[0],x[4]),stockHist[symbol])
for symbol2 in dupeList:
symbol2Arry=[]
if symbol2 == symbol:
savePair=0
continue
#symbol2Arry = stockHist[symbol2] #map(lambda x: x['Close'],db.nasdaq.find({'Symbol':symbol2},{'Close': 1, '_id':0 }))
symbol2Arry=map(lambda x:(x[0],x[4]),stockHist[symbol2])
minLen=min(len(symbolArry),len(symbol2Arry))
if minLen>0:
symbolArry=symbolArry[-minLen:]
symbol2Arry=symbol2Arry[-minLen:]
savePair=1
#if len(symbolArry)==len(symbol2Arry):
# savePair=1
if savePair:
# Need to create another dictionary that stores the average and standard deviation
# Also, need to keep track of correlation
# For each day we need to divide
symbol2dict=dict(symbol2Arry)
doPairAppend=pairsHistory.append
for day in symbolArry:
dayDate=day[0]
dayClose=day[1]
if dayDate not in symbol2dict:
continue
day2Close=symbol2dict[dayDate]
doPairAppend((dayDate,log(round(np.divide(float(dayClose),float(day2Close)),5))))
#print pairsHistory
#print symbol,symbol2
#raw_input()
pairsDict[("%s %s")%(symbol,symbol2)]=pairsHistory
if False:
for day in symbolArry:
dayDate=day[0]
day2Close=day[1]
#The date is stored in GMT time, so sometimes it has standard time and sometimes it's daylight
#time. Have to remove the GMT portion. Not sure if in import or when creating history.
if dayDate not in symbol2dict:
continue
dayClose=symbol2dict[dayDate]
pairsHistory.append((dayDate,round(np.divide(day2Close,dayClose),5)))
pairsDict[("%s %s")%(symbol2,symbol)]=pairsHistory
#pairsHistory=np.divide(symbol2Arry,symbolArry,)
#pairsDict[("%s %s")%(symbol2,symbol)]=pairsHistory
pairsHistory=[]
symbol1Arry=[]
symbol2Arry=[]
savePair=0
#if len(dupeList):
# dupeList.pop(0)
#else:
# break
pickle.dump( pairsDict, open(pairsFile, "wb" ))
return pairsDict
# Theunissen Lab dependencies.
from lasp.signal import lowpass_filter, highpass_filter
from lasp.sound import spectrogram, plot_spectrogram
from ipywidgets import *
# Specify the input file again. The segemented file is a pickle file that has s single block but
# multiple labelled segements corresponding to sections with song. It was created with audiosegmentnix.py (in songephys).
experiment_dir = "/auto/tdrive/billewood/intan data/LbY6074"
pkl_file_in = "LbY6074__161215_132633_seg_c1.pkl"
#pkl_file_out = "LbY6074__161216_104808_seg_c1.pkl" # c1 for check 1. You could use the same name and it will overwrite
# Open the Inupt Pkl file
pklFileIn = open(os.path.join(experiment_dir, 'NIX', pkl_file_in), 'rb')
# Read the data
block = pickle.load(pklFileIn)
pklFileIn.close()
print 'Found', len(block.segments), 'segments:'
nstudent = 0
ntutor = 0
for i, seg in enumerate(block.segments):
if seg.name.startswith('student'):
nstudent += 1
elif seg.name.startswith('tutor'):
ntutor += 1
print '\t', nstudent, 'student songs and', ntutor, 'tutor songs.'
# Add a little time before and after each event.
# times before and after were already added before segmentation
def load_pickle(name):
with open(name, 'wb') as p:
return P.load(p)
def load(self, fileobject=None):
fileobject, close = file_open(fileobject or self._cachefile, 'rb')
self._symcache = pickle.load(fileobject)
if close:
fileobject.close()
#!/usr/bin/env Python
try:
import cpickle as pickle
except ImportError:
import pickle
from collections import defaultdict
from nltk.corpus import brown, cmudict
FILENAME = 'dictionary.pickle'
try:
Dictionary = pickle.load(open(FILENAME))
except IOError:
cmudict = cmudict.dict()
BROWN_CATEGORIES = [
'lore',
'learned',
'fiction',
'mystery',
'science_fiction',
'adventure',
'romance',
'humor',
]
VOWELS = [
def load_table(filepath):
with open(filepath, 'rb') as f:
table_data = pickle.load(f)
return Table(table_data[0], table_data[1], table_data[2], table_data[3])
