def generate_roadway():
import julia
j = julia.Julia()
j.using("NGSIM")
base_dir = os.path.expanduser('~/Autoenv/data/')
j.write_roadways_to_dxf(base_dir)
j.write_roadways_from_dxf(base_dir)
def _call_powermodels(pm, julia_file): #pragma: no cover
buffer_file = os.path.join(tempfile.gettempdir(), "pp_pm.json")
logger.debug("writing PowerModels data structure to %s" % buffer_file)
with open(buffer_file, 'w') as outfile:
json.dump(pm, outfile)
try:
import julia
from julia import Main
except ImportError:
raise ImportError(
"Please install pyjulia to run pandapower with PowerModels.jl")
try:
j = julia.Julia()
except:
raise UserWarning(
"Could not connect to julia, please check that Julia is installed and pyjulia is correctly configured"
)
Main.include(os.path.join(pp_dir, "opf", 'pp_2_pm.jl'))
try:
run_powermodels = Main.include(julia_file)
except ImportError:
raise UserWarning("File %s could not be imported" % julia_file)
result_pm = run_powermodels(buffer_file)
return result_pm
def install(
julia_download_path='https://julialang-s3.julialang.org/bin/linux/x64/1.4/julia-1.4.0-linux-x86_64.tar.gz',
julia_target_path=None):
'''
:param julia_download_path: Path for the julia download, you can modify for your preferred version
:param julia_target_path: Specify where to install Julia, if not specified will install in $HOME$/julia
'''
if julia_target_path == None:
julia_target_path = os.path.join(os.path.expanduser("~"), 'julia')
if not os.path.isdir(julia_target_path):
os.mkdir(julia_target_path)
download_path = os.path.join(julia_target_path, 'julia_install.tar.gz')
print("Downloading Julia:")
wget.download(julia_download_path, download_path)
print("\nExtracting...")
tar = tarfile.open(download_path, "r:gz")
tar.extractall(julia_target_path)
_, partial_path = get_julia_path_from_dir(julia_target_path)
os.environ["PATH"] += os.pathsep + partial_path
os.system("echo '# added by dpmmpython' >> ~/.bashrc")
os.system("echo 'export PATH=\"" + partial_path + ":$PATH\"' >> ~/.bashrc")
print("Configuring PyJulia")
julia.install()
julia.Julia(compiled_modules=False)
print("Adding DPMMSubClusters package")
from julia import Pkg
Pkg.add("DPMMSubClusters")
print(
"Please exit the shell and restart, before attempting to use the package"
)
class Environment:
envname = "env"
jl = julia.Julia()
pacman = Pkg
def __init__(self, envname):
"""The initialization Function loads a virtual environment."""
print("Loading Package Manager")
self.pacman = Pkg
print("Pkg loaded")
if envname != "Global":
self.envname = envname
self.pacman.activate(envname)
print("Activated Package Environment: " + envname)
print("Environment available at: /" + envname)
def Add(self, pack):
"""Adds a package"""
self.pacman.add(pack)
def Using(self, packagename):
"""Imports and returns a package"""
impstring = "from julia import " + str(packagename)
exec(impstring)
grabstring = str(packagename)
w = eval(grabstring)
return (w)
def Switch(self, envname):
"""Allows you to quickly switch environments"""
self.pacman.activate(envname)
print("Environment created at /" + envname)
def List(self):
"""Lists the current packages inside of this environment"""
return (Pkg.installed())
def Remove(self, packagename):
"""Removes a given package from the environment."""
self.pacman.rm(packagename)
print("Package " + packagename + " Has been successfully removed.")
def Build(self, packagename):
self.pacman.build(packagename)
def Update(self):
self.pacman.update()
def Instantiate(self):
self.pacman.instantiate()
def Test(self, packagename):
self.pacman.test(packagename)
def Resolve(self):
self.pacman.resolve()
def Include(self, filepath):
self.jl.include(filepath)
print("You will now be able to add your file with Environment.add()")
def __makeJulia(self):
# make a dictionary that has all of the values that the .frac file gave you
# then make a julia file
with open(self.fracfile, 'r') as fracFile:
JuliaDict = dict()
infoName = ''
info = ''
count = 0
for i in fracFile:
if ':' in i:
infoName = i.split(':')[0]
infoName = infoName.strip('\n')
infoName = infoName.lower()
info = i.split(':')[1]
info = info.strip('\n')
info = info.strip(' ')
if infoName == 'pixels' or infoName == 'iterations' or infoName == 'Iterations':
if infoName == 'Iterations':
infoName = 'iterations'
JuliaDict[infoName] = int(info)
elif infoName != 'type':
JuliaDict[infoName] = float(info)
else:
JuliaDict[infoName] = info
julia.Julia(JuliaDict, self.__fileName)
def load_julia_with_Shearlab():
# Importing base
j = julia.Julia() #, debug='False')
j.eval('using Shearlab')
j.eval('using PyPlot')
j.eval('using Images')
return j
def __init__(self, counter, max_size, force_python=False):
self.cache = dict()
if not force_python:
try:
import julia
self.julia = julia.Julia()
self.julia.using("Revtok")
self.vocab = self.julia.buildvocab(counter, max_size)
return
except ImportError:
print('For faster subwords, please install Julia 0.6, pyjulia, '
'and Revtok.jl. Falling back to Python implementation...')
except Exception as e:
print(e)
print('for faster subwords, please install Revtok.jl. '
'Falling back to Python implementation...')
self.vocab = Counter(''.join(counter.keys())).most_common()
self.vocab.sort(key=lambda tup: (-tup[1], tup[0]))
self.vocab = dict(self.vocab)
ngrams = list(NGrams(counter).ngrams.values())
ngrams.sort(key=attrgetter('text'))
key = attrgetter('entropy')
for i in tqdm(range(max_size - len(self.vocab)), 'building subword vocab'):
ngrams.sort(key=key, reverse=True)
best = ngrams[0]
#print(best)
for utterance in best.utterances:
seen = set([best])
for ngram in utterance.ngrams:
if ngram not in seen:
ngram.count -= utterance.count * utterance.overlaps(ngram, best)
seen.add(ngram)
self.vocab[ngrams[0].text] = ngrams[0].entropy
ngrams = ngrams[1:]
self.julia = None
def __init__(
self,
n_features, # number of features
batch_size, # number of simultaneous sims
weights, # reward weights, as a vector
# NGSIM trajdata indeces, full set is [1,2,3,4,5,6]
trajdata_indeces,
max_nsteps=1000,
):
self.tstep = 0
self.max_nsteps = max_nsteps
self.n_features = n_features
# Load in functions
self._name_dump = 'julia(JuliaDriveEnv2DBatch.auto2D_path={})'.format(
auto2D_path)
#debug = open('debug.log', 'a'); debug.write('{}\n'.format(self._name_dump)); debug.close()
self.j = julia.Julia()
self.j.eval("include(\"" + auto2D_path + "\")")
self.j.using("Auto2D")
ipdb.set_trace()
self.simparams = self.j.gen_simparams(trajdata_indeces, weights[0],
weights[1], weights[2],
weights[3], weights[4],
weights[5], weights[6],
batch_size)
self.features = self.j.alloc_features(batch_size)
if GX:
_, self.ax = plt.subplots(1, 1)
def __init__(self, nbArms, lower=0., amplitude=1.):
""" Will fail directly if the bridge with julia is unavailable or buggy."""
super(UCBjulia, self).__init__(nbArms,
lower=lower,
amplitude=amplitude)
self.t = 0
# Importing the julia module and creating the bridge
try:
import julia
except ImportError:
print(
"Error: unable to load the 'julia' Python module. Install with 'pip install julia', or see https://github.com/JuliaPy/pyjulia/"
) # DEBUG
_j = julia.Julia()
try:
self._index_function = _j.evalfile("Policies/UCBjulia.jl")
except RuntimeError:
try:
self._index_function = _j.evalfile("UCBjulia.jl")
except RuntimeError:
raise ValueError(
"Error: Unable to load 'UCBjulia.jl' julia file."
) # WARNING
try:
self._index_function([1], [1], 1, 1)
except (RuntimeError, ValueError):
raise ValueError(
"Error: the index function loaded from 'UCBjulia.jl' is bugged or unavailable."
) # WARNING
def _call_powermodels(pm, julia_file=None):
buffer_file = os.path.join(tempfile.gettempdir(), "pp_pm.json")
with open(buffer_file, 'w') as outfile:
json.dump(pm, outfile)
try:
import julia
except ImportError:
raise ImportError(
"Please install pyjulia to run pandapower with PowerModels.jl")
try:
j = julia.Julia()
except:
raise UserWarning(
"Could not connect to julia, please check that Julia is installed and pyjulia is correctly configured"
)
if julia_file is None:
folder = os.path.join(pp_dir, "opf")
julia_file = os.path.join(folder, 'run_powermodels.jl')
try:
run_powermodels = j.include(julia_file)
except:
raise UserWarning("File %s could not be imported" % julia_file)
result_pm = run_powermodels(buffer_file)
return result_pm
def __init__(
self,
n_features, # number of features
batch_size, # number of simultaneous sims
weights, # reward weights, as a vector
# NGSIM trajdata indeces, full set is [1,2,3,4,5,6]
trajdata_indeces,
max_nsteps=1000,
):
self.tstep = 0
self.max_nsteps = max_nsteps
self.n_features = n_features
# Load in functions
self.j = julia.Julia()
self.j.eval("include(\"" + auto2D_path + "\")")
self.j.using("Auto2D")
self.simparams = self.j.gen_simparams(trajdata_indeces, weights[0],
weights[1], weights[2],
weights[3], weights[4],
weights[5], weights[6],
batch_size)
self.features = self.j.alloc_features(batch_size)
if GX:
_, self.ax = plt.subplots(1, 1)
def add_procs(procs_count):
j = julia.Julia()
j.eval('using Distributed')
j.eval('addprocs(' + str(procs_count) + ')')
j.eval('@everywhere using DPMMSubClusters')
j.eval('@everywhere using LinearAlgebra')
j.eval('@everywhere BLAS.set_num_threads(2)')
def load_julia_with_Shearlab():
"""Function to load Shearlab."""
# Importing base
j = julia.Julia()
j.eval('using Shearlab')
j.eval('using PyPlot')
j.eval('using Images')
return j
def __init__(self):
# Load in functions
self._name_dump = 'julia(JuliaLQGEnv.LQG_path={})'.format(LQG_path)
#debug = open('debug.log', 'a'); debug.write('{}\n'.format(self._name_dump)); debug.close()
self.j = julia.Julia()
self.j.eval("include(\"" + LQG_path + "\")")
self.j.using("juliaLQG")
def main():
j = julia.Julia();
j.include("julia_POMCP_Controller.jl");
makeSolver = j.eval("makeSolver");
[solver,pomdp,planner] = makeSolver();
watch(j,solver,pomdp,planner);
def __init__(self,
n_features,
trajdata_indeces,
train_seg_index=0,
frame_num=0,
save_history=False):
self.t = 0.1
self.tstep = 0
self.max_nsteps = 1000
self.n_features = n_features
self.train_seg_index = train_seg_index
self.frame_num = frame_num
# Load in functions
self._name_dump = 'julia(JuliaDriveEnv2D.auto2D_path={})'.format(
auto2D_path)
#debug = open('debug.log', 'a'); debug.write('{}\n'.format(self._name_dump)); debug.close()
self.j = julia.Julia()
self.j.eval("include(\"" + auto2D_path + "\")")
self.j.using("Auto2D")
#self.j.set_TRAJDATAS(tj_ix= tj_ix)
#self.simparams = self.j.gen_simparams([1])
ipdb.set_trace()
if trajdata_indeces == []:
print "USING PASSIVE/AGGRESSIVE"
def append_path(x):
return pulltraces_path + x
trajdatas = [
"trajdata_passive_aggressive1.txt",
"trajdata_passive_aggressive2.txt"
]
roadways = [
"roadway_passive_aggressive.txt",
"roadway_passive_aggressive.txt"
]
# self.simparams= self.j.gen_simparams_from_trajdatas(map(append_path,trajdatas),map(append_path,roadways),
# weights[0], weights[1], weights[2], weights[3], weights[4],
# weights[5])
ipdb.set_trace()
self.simparams = self.j.gen_simparams_from_trajdatas(
map(append_path, trajdatas), map(append_path, roadways),
type_gru)
else:
#self.simparams = self.j.gen_simparams(trajdata_indeces, weights[0], weights[1], weights[2], weights[3], weights[4], weights[5])
ipdb.set_trace()
self.simparams = self.j.gen_simparams(trajdata_indeces, type_gru)
self.features = self.j.alloc_features()
if GX:
_, self.ax = plt.subplots(1, 1)
def get_statistics(simfile):
jl = julia.Julia()
jl.eval("using Mamba")
jl.eval("include(\"utils.jl\")")
jl.eval("include(\"../src/celerite.jl\")")
if not jl.eval("@isdefined CeleriteDist"):
jl.eval("eval(DefineCeleriteDist())")
sim = jl.eval("sim = read(\"{0}\", ModelChains)".format(simfile))
return jl.eval("summarystats(sim)")
def crawl_data(self, script_name):
"""
Runs a Julia script to crawl data.
:param str script_name: Name of Julia script
:return:
"""
jul = julia.Julia(compiled_modules=False)
jul.include(script_name)
def __init__(self):
self.dt = 0.1
# Load in functions
self.j = julia.Julia()
self.j.eval("include(\"" + auto1D_path + "\")")
self.j.using("Auto1D")
self.roadway = self.j.gen_stadium_roadway(1) # one-lane track
def get_autocor(simfile, lags):
lags = iterable_to_string(lags)
jl = julia.Julia()
jl.eval("using Mamba")
jl.eval("include(\"utils.jl\")")
jl.eval("include(\"../src/celerite.jl\")")
if not jl.eval("@isdefined CeleriteDist"):
jl.eval("eval(DefineCeleriteDist())")
sim = jl.eval("sim = read(\"{0}\", ModelChains)".format(simfile))
return jl.eval("autocor(sim, lags={0})".format(lags))
def task3(task1out, task2out):
if (data_json['steps']['task3']['in']['files']['source']):
if (data_json['steps']['task3']['in']['files']['script'] == 'julia'):
j = julia.Julia()
j.include(data_json['steps']['task3']['in']['files']['name'])
r4 = j.include(data_json['steps']['task3']['in']['files']['name2'])
outfile = j.filesmeth(task1out, task2out)
return outfile
else:
return (print("Error in Task3"))
def _call_pandamodels(buffer_file, julia_file, dev_mode): # pragma: no cover
try:
import julia
from julia import Main
from julia import Pkg
from julia import Base
except ImportError:
raise ImportError(
"Please install pyjulia properly to run pandapower with PandaModels.jl."
)
try:
julia.Julia()
except:
raise UserWarning(
"Could not connect to julia, please check that Julia is installed and pyjulia is correctly configured"
)
if not Base.find_package("PandaModels"):
logger.info(
"PandaModels.jl is not installed in julia. It is added now!")
Pkg.Registry.update()
Pkg.add("PandaModels")
if dev_mode:
logger.info("installing dev mode is a slow process!")
Pkg.resolve()
Pkg.develop("PandaModels")
# add pandamodels dependencies: slow process
Pkg.instantiate()
Pkg.build()
Pkg.resolve()
logger.info("Successfully added PandaModels")
if dev_mode:
Pkg.develop("PandaModels")
Pkg.build()
Pkg.resolve()
Pkg.activate("PandaModels")
try:
Main.using("PandaModels")
except ImportError:
raise ImportError("cannot use PandaModels")
Main.buffer_file = buffer_file
result_pm = Main.eval(julia_file + "(buffer_file)")
# if dev_mode:
# Pkg.activate()
# Pkg.free("PandaModels")
# Pkg.resolve()
return result_pm
def __init__(self):
self.dt = 0.1
# Load in functions
self._name_dump = 'julia(JuliaDriveEnv.auto1D_path={})'.format(
auto1D_path)
#debug = open('debug.log', 'a'); debug.write('{}\n'.format(self._name_dump)); debug.close()
self.j = julia.Julia()
self.j.eval("include(\"" + auto1D_path + "\")")
self.j.using("Auto1D")
self.roadway = self.j.gen_stadium_roadway(1) # one-lane track
def test_julia_connection():
try:
import julia
except:
raise ImportError(
"install pyjulia properlly to run PandaModels.jl")
try:
julia.Julia()
except:
raise UserWarning(
"cannot connect to julia, check pyjulia configuration")
def get_sim(simfile):
jl = julia.Julia()
jl.eval("using Mamba")
jl.eval("include(\"utils.jl\")")
jl.eval("include(\"../src/celerite.jl\")")
if not jl.eval("@isdefined CeleriteDist"):
jl.eval("eval(DefineCeleriteDist())")
#jl.eval(jl.DefineCeleriteDist())
sim = jl.eval("sim = read(\"{0}\", ModelChains)".format(simfile))
sim = jl.eval("sim.value")
labels = jl.eval("sim.names")
return sim, labels
def make_julia():
j = julia.Julia(
make_mandelbrot(),
canvas,
allowed_keyevents=general_keys + [
pygame.K_w, # move julia constant up
pygame.K_a, # move julia constant left
pygame.K_s, # move julia constant down
pygame.K_d, # move julia constant right
])
j.view.rectify()
return j
def julia(self):
if isinstance(self._julia, Exception):
return None
elif self._julia is None:
try:
self._julia = julia.Julia()
except Exception as e:
print(
"Creating julia.Julia raised an error - falling back to slower alternative."
)
self._julia = e
return None
return self._julia
def __init__(self, env_id, env_params, using):
# initialize julia
self.j = julia.Julia()
self.j.eval('include(\"{}\")'.format(
os.path.expanduser('~/.juliarc.jl')))
self.j.using(using)
# initialize environment
self.env = self.j.make(env_id, env_params)
self._observation_space = build_space(
*self.j.observation_space_spec(self.env))
self._action_space = build_space(*self.j.action_space_spec(self.env))
def __init__(self,
network=None,
validation=None,
selectedQueryMode=None,
selectedPredictionMode=None,
method='AptRank',
param=dict()):
'''
Constructor:
Calls the parent constructor
'''
if validation is None:
self.network = network
self.validation = validation
else:
try:
method = method + str(param['diff_type'])
except:
method = method + '1'
for k in param:
if k == 'K' or k == 'S' or k == 'lowerBoundFactor':
method = method + '_' + str(k) + str(param[k])
AlgorithmParent.__init__(
self,
method=method,
network=network,
validation=validation,
selectedQueryMode=selectedQueryMode,
selectedPredictionMode=selectedPredictionMode)
self.j = julia.Julia()
self.alpha = None
# K - the number of terms used in the APTRANK, default is 8
self.K = 8
# S - the number of independent experiments during training,
# default is 5
self.S = 5
# ratio - the split ratio between fiiting set and validation
# set, default is 0.8
self.ratio = 0.5
# diff_type - choose what kind of diffusion matrix you want, 1 for G/rho,
# 2 for G*D^{-1}, 3 for (D - G)/rho and 4 for (I - D^{-1/2}*G*D^{-1/2})/norm(L,1),
# where rho is the spectral radius and D is the out-degree of each
# vertex, defalut is 1
self.diff_type = 1
# sampling_type- 1 for randomly sampling using ratio, 2 for S-fold
# cross validation, default is 1
self.sampling_type = 1
self.lowerBoundFactor = 2
self.__dict__.update(param)
self.lowerBound = 1.0 / ((self.K - 1)**self.lowerBoundFactor)
def __init__(self, env_dict = {}, n_egos = 1, z_dim = 0, normalize_obs = True, prior_type = "discrete",
end_on_failure = True, index_features = False):
self.j = julia.Julia()
include_statement = "include(\"{}\")".format(AUTO2D_PATH)
self.j.eval(include_statement)
self.j.using("Auto2D")
self.simparams = self.j.gen_simparams(env_dict)
self.mix_class = env_dict["mix_class"]
self.model_all = env_dict["model_all"]
self.domain_indices = env_dict["domain_indices"]
self.index_features = index_features
self.z_dim = z_dim
self._prior_f = None
self._n_egos = n_egos
self.prior_type = prior_type
self.end_on_failure = end_on_failure
# pull observation mean and std for normalization
with h5py.File("{}/{}".format(PROJECT_PATH,normalizer_paths["ORIG"]),"r") as hf:
self.obs_mean = \
np.repeat(hf['obs_mean'][...],self.n_egos)
self.obs_std = \
np.repeat(hf['obs_std'][...],self.n_egos)
self.act_mean = \
np.repeat(hf['act_mean'][...],self.n_egos)
self.act_std = \
np.repeat(hf['act_std'][...], self.n_egos)
if not normalize_obs:
self.obs_mean = np.zeros_like(self.obs_mean)
self.obs_std = np.ones_like(self.obs_std)
# controls latent label
if self.z_dim > 0:
if prior_type == "standard": # standard gaussian prior
self.z_mean = np.zeros(self.z_dim)[None,...]
self.z_std = np.ones(self.z_dim)[None,...]
elif prior_type == "discrete":
self.z_mean, self.z_std = np.zeros(self.z_dim),np.zeros(self.z_dim)
else:
self.z = []
obsdim = self.j.get_obsdim(self.simparams)
_, self.ax = plt.subplots(1,1)
plt.ion()