def _attempt_listening(self):
self._sync_actions()
if self._awaiting_connection is not None:
# Cannot handle multiple connection requests at the same time
return
try:
socketutils.wait_for_read([self._listener_sock], timeout=0)
except socketutils.TimeoutError:
return
conn = Bunch()
data, conn.remote_address = self._listener_sock.recvfrom(MAX_PACKET_SIZE)
remote_hostname, remote_port = conn.remote_address
remote_version, conn.remote_host_id, conn.remote_host_name = loads(data)
assert remote_version == self.protocol_version, "Attempt to connect with client of different version"
if conn.remote_address in self._already_connected:
# We may be receiving a retransmit of the connection
# packet from before, anyhow, he is already connected, so
# ignore it.
return
self.notify("new_connector1", conn.remote_host_name, conn.remote_address)
conn.sock=socketutils.new_udp_socket()
conn.sock.connect(conn.remote_address)
host_ids = [(host.id, host.name) for host in self.hosts]
welcome_to_send = dumps(('WELCOME!', (self._publicized_data, host_ids)))
conn._welcome_to_send = welcome_to_send
conn._welcome_count = 0
self._awaiting_connection = conn
self.update = self._send_welcome
def __delattr__(self, key):
"""od.__delitem__(y) <==> del od[y]"""
# Deleting an existing item uses self._map to find the link which is
# then removed by updating the links in the predecessor and successor nodes.
Bunch.__delattr__(self,key)
link_prev, link_next, key = self._map.pop(key)
link_prev[1] = link_next
link_next[0] = link_prev
def __setattr__(self, key, value):
"""od.__setitem__(i, y) <==> od[i]=y"""
# Setting a new item creates a new link which goes at the end of the linked
# list, and the inherited dictionary is updated with the new key/value pair.
if not hasattr(self,key) and not hasattr(self.__class__,key):
root = dict.__getitem__(self,'_root')
last = root[0]
map = dict.__getitem__(self,'_map')
last[1] = root[0] = map[key] = [last, root, key]
Bunch.__setattr__(self,key, value)
def clear(self):
"""od.clear() -> None. Remove all items from od."""
try:
for node in self._map.itervalues():
del node[:]
root = self._root
root[:] = [root, root, None]
self._map.clear()
except AttributeError:
pass
Bunch.clear(self)
def make_control(argv):
# return a Bunch
print argv
if not(2 <= len(argv) <= 3):
usage('invalid number of arguments')
pcl = ParseCommandLine(argv)
arg = Bunch(
base_name='valgbr',
yyyymm=argv[1],
test=pcl.has_arg('--test'),
)
try:
arg.yyyymm = int(arg.yyyymm)
except:
usage('YYYYMM not an integer')
random_seed = 123
random.seed(random_seed)
dir_working = Path().dir_working()
debug = False
out_file_name = (
('test-' if arg.test else '') +
'%s.pickle' % arg.yyyymm
)
# assure output directory exists
dir_path = dir_working + arg.base_name + '/'
if not os.path.exists(dir_path):
os.makedirs(dir_path)
fixed_hps = Bunch(
loss='quantile',
alpha=0.5,
n_estimators=1000,
max_depth=3,
max_features=None)
return Bunch(
arg=arg,
debug=debug,
fixed_hps=fixed_hps,
path_in=dir_working + 'samples-train.csv',
path_out=dir_path + out_file_name,
random_seed=random_seed,
test=arg.test,
)
def make_control(argv):
# return a Bunch
print argv
if not(4 <= len(argv) <= 7):
usage('invalid number of arguments')
pcl = ParseCommandLine(argv)
arg = Bunch(
base_name='ege',
folds=pcl.get_arg('--folds'),
rfbound=pcl.get_arg('--rfbound'), # arg.rbound is None or a string or a list of strings
test=pcl.has_arg('--test'),
)
if arg.rfbound is not None:
if not len(arg.rfbound) == 2:
usage('corret is --rfbound HP YYYYMM')
arg.hp = arg.rfbound[0]
arg.yyyymm = arg.rfbound[1]
if not(arg.hp in ('max_depth', 'max_features')):
usage('--rfbound {max_depth|max_features} YYYYMM')
if arg.folds is None:
usage('--folds is required')
else:
arg.folds = int(arg.folds)
random_seed = 123
random.seed(random_seed)
dir_working = Path().dir_working()
debug = False
out_file_name_base = (
('test-' if arg.test else '') +
arg.base_name +
('' if arg.rfbound is None else '-rfbound-%s-%s' % (arg.hp, arg.yyyymm)) +
('-folds-%02d' % arg.folds)
)
return Bunch(
arg=arg,
debug=debug,
path_in=dir_working + 'samples-train-validate.csv',
path_out=dir_working + out_file_name_base + '.pickle',
random_seed=random_seed,
test=arg.test,
)
def make_control(argv):
# return a Bunch
print argv
if not(4 <= len(argv) <= 5):
usage('invalid number of arguments')
pcl = ParseCommandLine(argv)
arg = Bunch(
base_name='rfbound',
hp=argv[1],
yyyymm=argv[2],
folds=argv[3],
test=pcl.has_arg('--test'),
)
try:
arg.folds = int(arg.folds)
except:
usage('INT not an integer; ' + str(arg.folds))
random_seed = 123
random.seed(random_seed)
dir_working = Path().dir_working()
debug = False
out_file_name = (
'%s/%s%s-%s-folds-%02d.pickle' % (
arg.base_name,
('test-' if arg.test else ''),
arg.hp,
arg.yyyymm,
arg.folds)
)
# assure the output directory exists
dir_path = dir_working + arg.base_name
if not os.path.exists(dir_path):
os.makedirs(dir_path)
return Bunch(
arg=arg,
debug=debug,
path_in=dir_working + 'samples-train-validate.csv',
path_out=dir_working + out_file_name,
random_seed=random_seed,
test=arg.test,
)
def lookup_rpcsvc(key):
"""Get the send and receive program numbers for a service
looked up by a service string. May raise a KeyError."""
recv, send = ProgramNumbers[key]
return Bunch(server_send_prgnum=send,
server_receive_prgnum=recv,
client_send_prgnum=recv,
client_receive_prgnum=send)
def test_DealFromBunch_DealMoreTilesThanInBunch_ReturnsWholeBunch(self) :
# Arrange
tiles = []
tileA = Tile('A', 1, 1, 1)
tileB = Tile('B', 1, 1, 1)
tiles.append(tileA)
tiles.append(tileB)
toDeal = 10
expected = 2
bunch = Bunch(tiles)
# Act
resultingBunch = bunch.DealFromBunch(toDeal)
result = len(resultingBunch)
# Assert
self.assertEqual(result, expected)
def initialize(atoms):
"Assign parameters for the rest of the calculation"
from Slater import gauss_powers,gexps,gcoefs,s_or_p
from MINDO3_Parameters import Uss,Upp,IPs,IPp,CoreQ,f03,nbfat,\
zetas,zetap,Eat,Hfat,gss,gsp,gpp,gppp,hsp,hppp,NQN
from CGBF import CGBF
from Bunch import Bunch # Generic object to hold basis functions
ibf = 0 # Counter to overall basis function count
for atom in atoms:
xyz = atom.pos()
atom.Z = CoreQ[atom.atno]
atom.basis = []
atom.rho = e2/f03[atom.atno]
atom.nbf = nbfat[atom.atno]
atom.Eref = Eat[atom.atno]
atom.Hf = Hfat[atom.atno]
atom.gss = gss[atom.atno]
atom.gsp = gsp[atom.atno]
atom.gpp = gpp[atom.atno]
atom.gppp = gppp[atom.atno]
atom.hsp = hsp[atom.atno]
atom.hppp = hppp[atom.atno]
for i in xrange(atom.nbf):
bfunc = Bunch()
atom.basis.append(bfunc)
bfunc.index = ibf # pointer to overall basis function index
ibf += 1
bfunc.type = i # s,x,y,z
bfunc.atom = atom # pointer to parent atom
bfunc.cgbf = CGBF(xyz,gauss_powers[i])
zi = gexps[(NQN[atom.atno],s_or_p[i])]
ci = gcoefs[(NQN[atom.atno],s_or_p[i])]
if i:
zeta = zetap[atom.atno]
bfunc.u = Upp[atom.atno]
bfunc.ip = IPp[atom.atno]
else:
zeta = zetas[atom.atno]
bfunc.u = Uss[atom.atno]
bfunc.ip = IPs[atom.atno]
for j in xrange(len(zi)):
bfunc.cgbf.add_primitive(zi[j]*zeta*zeta,ci[j])
bfunc.cgbf.normalize()
return atoms
def make_control(argv):
# return a Bunch
print argv
if not(2 <= len(argv) <= 3):
usage('invalid number of arguments')
pcl = ParseCommandLine(argv)
arg = Bunch(
base_name='linval',
yyyymm=argv[1],
test=pcl.has_arg('--test'),
)
try:
arg.yyyymm = int(arg.yyyymm)
except:
usage('YYYYMM not an integer')
random_seed = 123
random.seed(random_seed)
dir_working = Path().dir_working()
debug = False
out_file_name = (
('test-' if arg.test else '') +
'%s.pickle' % arg.yyyymm
)
# assure output directory exists
dir_path = dir_working + arg.base_name + '/'
if not os.path.exists(dir_path):
os.makedirs(dir_path)
return Bunch(
arg=arg,
debug=debug,
path_in=dir_working + 'samples-train-validate.csv',
path_out=dir_path + out_file_name,
random_seed=random_seed,
test=arg.test,
)
def make_control(argv):
# return a Bunch
print argv
if len(argv) not in (1, 2, 3):
usage('invalid number of arguments')
pcl = ParseCommandLine(argv)
arg = Bunch(
base_name='chart-05',
data=pcl.has_arg('--data'),
test=pcl.has_arg('--test'),
)
if len(argv) == 3:
both = arg.data and arg.test
if not both:
usage('there is an extra invocation option')
random_seed = 123
random.seed(random_seed)
dir_working = Path().dir_working()
debug = False
reduced_file_name = ('test-' if arg.test else '') + 'data.pickle'
# assure output directory exists
dir_path = dir_working + arg.base_name + '/'
if not os.path.exists(dir_path):
os.makedirs(dir_path)
return Bunch(
arg=arg,
debug=debug,
path_in_ege=dir_working + 'valgbr/*.pickle',
path_reduction=dir_path + reduced_file_name,
path_chart_base=dir_path,
random_seed=random_seed,
test=arg.test,
)
def get_floaters(hostbnches):
"""Map floating interface names to generic node names."""
res = Bunch()
for key in hostbnches.keys():
try:
alt_ifcs = hostbnches[key]['alt']
for name in alt_ifcs:
res[name] = key
except KeyError:
continue
return res
def make_control(argv):
# return a Bunch
print argv
if len(argv) not in (1, 2):
usage('invalid number of arguments')
pcl = ParseCommandLine(argv)
if pcl.has_arg('--help'):
usage()
arg = Bunch(
base_name=argv[0].split('.')[0],
test=pcl.has_arg('--test'),
)
random_seed = 123456
random.seed(random_seed)
path = Path() # use the default dir_input
debug = False
file_out_transactions = (('testing-' if arg.test else '') + arg.base_name +
'-al-g-sfr' + '.csv')
return Bunch(
arg=arg,
debug=debug,
max_sale_price=85e6, # according to Wall Street Journal
path=path,
path_in_census_features=path.dir_working() +
'census-features-derived.csv',
path_in_parcels_features_census_tract=path.dir_working() +
'parcels-features-census_tract.csv',
path_in_parcels_features_zip5=path.dir_working() +
'parcels-features-zip5.csv',
path_out_transactions=path.dir_working() + file_out_transactions,
random_seed=random_seed,
test=arg.test,
)
def __init__(self, timestep, totaltime, intmethod, ElectricF, MagneticF,
BunchSize):
self.timestep = timestep
self.totaltime = totaltime
self.intmethod = intmethod
self.ElectricF = ElectricF
self.MagneticF = MagneticF
self.BunchSize = BunchSize
Field = EMField(Magnetic=self.MagneticF, Electric=self.ElectricF)
self.Field = Field
ParticleBunch = Bunch(self.BunchSize)
self.Bunch = ParticleBunch
self.Bunch.CreateBunch()
def initialize(atoms):
"Assign parameters for the rest of the calculation"
from Slater import gauss_powers,gexps,gcoefs,s_or_p
from MINDO3_Parameters import Uss,Upp,IPs,IPp,CoreQ,f03,nbfat,\
zetas,zetap,Eat,Hfat,gss,gsp,gpp,gppp,hsp,hppp,NQN
from CGBF import CGBF
from Bunch import Bunch # Generic object to hold basis functions
ibf = 0 # Counter to overall basis function count
for atom in atoms:
xyz = atom.pos()
atom.Z = CoreQ[atom.atno]
atom.basis = []
atom.rho = e2/f03[atom.atno]
atom.nbf = nbfat[atom.atno]
atom.Eref = Eat[atom.atno]
atom.Hf = Hfat[atom.atno]
atom.gss = gss[atom.atno]
atom.gsp = gsp[atom.atno]
atom.gpp = gpp[atom.atno]
atom.gppp = gppp[atom.atno]
atom.hsp = hsp[atom.atno]
atom.hppp = hppp[atom.atno]
for i in range(atom.nbf):
bfunc = Bunch()
atom.basis.append(bfunc)
bfunc.index = ibf # pointer to overall basis function index
ibf += 1
bfunc.type = i # s,x,y,z
bfunc.atom = atom # pointer to parent atom
bfunc.cgbf = CGBF(xyz,gauss_powers[i])
zi = gexps[(NQN[atom.atno],s_or_p[i])]
ci = gcoefs[(NQN[atom.atno],s_or_p[i])]
if i:
zeta = zetap[atom.atno]
bfunc.u = Upp[atom.atno]
bfunc.ip = IPp[atom.atno]
else:
zeta = zetas[atom.atno]
bfunc.u = Uss[atom.atno]
bfunc.ip = IPs[atom.atno]
for j in range(len(zi)):
bfunc.cgbf.add_primitive(zi[j]*zeta*zeta,ci[j])
bfunc.cgbf.normalize()
return atoms
def _on_training_start(self, sess):
# Writers and savers
self.summary_writer = tf.summary.FileWriter(FLAGS.logdir, sess.graph)
self.saver = tf.train.Saver()
self._build_embedding_saver(sess)
self._restore_model(sess)
# Loss summaries
self._build_summaries()
self.epoch_stats = get_stats_template()
self.stats = Bunch(epoch_accuracy=[],
epoch_reconstructions=[],
permutation=None)
def __new__(klass,*args,**kwarg):
self = Bunch.__new__(klass)
try:
self._root
except:
root = [] # sentinel node
root[:] = [root, root, None]
dict.__setitem__(self,'_root',root)
dict.__setitem__(self,'_map' ,{})
return self
def __new__(klass, *args, **kwarg):
self = Bunch.__new__(klass)
try:
self._root
except:
root = [] # sentinel node
root[:] = [root, root, None]
dict.__setitem__(self, '_root', root)
dict.__setitem__(self, '_map', {})
return self
def convert_csv_to_yolo_format(in_dir, out_dir, gt_csv_file_path):
'''This Fn takes the input csv file and write the images and texts file as in yolo format.'''
counter = 0
total_roi_instances = 0
with open(gt_csv_file_path, 'r') as csvfile:
samples = csv.reader(csvfile, delimiter=',')
for sample in samples:
print('row count ', counter)
counter += 1
img_name = str(sample[0])
img_path = os.path.join(in_dir, img_name)
img_no = get_file_number(img_name)
img = cv2.imread(img_path)
no_instances = int(sample[1])
total_roi_instances += no_instances
lstIndex = 2
list_instances = []
for ii in range(no_instances):
x_min = float(sample[lstIndex])
lstIndex += 1
y_min = float(sample[lstIndex])
lstIndex += 1
x_max = float(sample[lstIndex])
lstIndex += 1
y_max = float(sample[lstIndex])
lstIndex += 1
# class_lbl = list_class_name.index(class_name)
class_lbl = int(sample[lstIndex])
lstIndex += 1
instance = Bunch(x_min=x_min,
y_min=y_min,
x_max=x_max,
y_max=y_max,
lbl=class_lbl)
list_instances.append(instance)
llist_yolo = bunch_2_yolo.get_yolo_llist(list_instances, img)
txt_path, txt_name = image_utils.get_txt_path_name_frm_index(
int(img_no), out_dir)
old_img_path = os.path.join(in_dir, img_name)
new_img_path, new_img_name = image_utils.get_img_path_name_frm_index(
int(img_no), out_dir)
copyfile(old_img_path, new_img_path)
f = open(txt_path, 'w')
writer = csv.writer(f, delimiter=' ')
for list_yolo in llist_yolo:
writer.writerow(list_yolo)
f.close()
print('total roi instances = ', total_roi_instances)
def make_control(argv):
# return a Bunch
print argv
if len(argv) not in (1, 2):
usage('invalid number of arguments')
pcl = ParseCommandLine(argv)
arg = Bunch(
base_name='chart-07',
data=pcl.has_arg('--data'),
test=pcl.has_arg('--test'),
)
random_seed = 123
random.seed(random_seed)
dir_working = Path().dir_working()
debug = False
reduced_file_name = ('test-' if arg.test else '') + 'data.pickle'
# assure output directory exists
dir_path = dir_working + arg.base_name + '/'
if not os.path.exists(dir_path):
os.makedirs(dir_path)
return Bunch(
arg=arg,
debug=debug,
path_in_best=dir_working + 'chart-06/best.pickle',
path_in_samples='../data/working/samples-train.csv',
path_reduction=dir_path + reduced_file_name,
path_chart_base=dir_path,
random_seed=random_seed,
test=arg.test,
)
def _on_epoch_finish(self, epoch, start_time, sess):
elapsed = time.time() - start_time
self.epoch_stats.total_loss = guard_nan(self.epoch_stats.total_loss)
accuracy = np.nan_to_num(
100000 * np.sqrt(self.epoch_stats.total_loss /
np.prod(self.batch_shape) / self.epoch_size))
# SAVE
if is_stopping_point(epoch, FLAGS.max_epochs, FLAGS.save_every):
self.saver.save(sess, self.get_checkpoint_path())
# VISUALIZE
if is_stopping_point(epoch, FLAGS.max_epochs, FLAGS.eval_every):
evaluation = self.evaluate(sess, take=FLAGS.visualiza_max)
data = {
'enc': np.asarray(evaluation.encoded),
'rec': np.asarray(evaluation.reconstructed),
'blu': np.asarray(evaluation.source)
}
error_info = '%d(%d.%d.%d)' % (
np.nan_to_num(accuracy),
np.nan_to_num(evaluation.loss) / evaluation.size,
np.nan_to_num(evaluation.eval_loss) / evaluation.size,
np.nan_to_num(evaluation.dumb_loss) / evaluation.size)
meta = Bunch(suf='encodings',
e='%06d' % int(self.get_past_epochs()),
er=error_info)
# print(data, meta.to_file_name(folder=FLAGS.save_path))
np.save(meta.to_file_name(folder=FLAGS.save_path), data)
vis.plot_encoding_crosssection(evaluation.encoded,
meta.to_file_name(
FLAGS.save_path, 'jpg'),
evaluation.source,
evaluation.reconstructed,
interactive=FLAGS.dev)
self._save_visualization_to_summary()
self.stats.epoch_accuracy.append(accuracy)
self._print_epoch_info(accuracy, epoch, FLAGS.max_epochs, elapsed)
if epoch + 1 != FLAGS.max_epochs:
self.epoch_stats = get_stats_template()
def make_control(argv):
# return a Bunch
print argv
if len(argv) not in (1, 2):
usage('invalid number of arguments')
pcl = ParseCommandLine(argv)
arg = Bunch(
base_name=argv[0].split('.')[0],
test=pcl.has_arg('--test'),
)
random_seed = 123
random.seed(random_seed)
dir_working = Path().dir_working()
debug = False
out_file_name_base = ('testing-' if arg.test else '') + arg.base_name
return Bunch(
arg=arg,
debug=debug,
fraction_test=0.2,
max_sale_price=85e6, # according to Wall Street Journal
path_in=dir_working + 'transactions-al-g-sfr.csv',
path_out_info_reasonable=dir_working + out_file_name_base +
'-info-reasonable.pickle',
path_out_test=dir_working + out_file_name_base + '-test.csv',
path_out_train=dir_working + out_file_name_base + '-train.csv',
path_out_train_validate=dir_working + out_file_name_base +
'-train-validate.csv',
path_out_validate=dir_working + out_file_name_base + '-validate.csv',
random_seed=random_seed,
test=arg.test,
)
def make_control(argv):
# return a Bunch
print argv
if len(argv) not in (3, 4):
usage('invalid number of arguments')
pcl = ParseCommandLine(argv)
arg = Bunch(
base_name=argv[0].split('.')[0],
geo=pcl.get_arg('--geo'),
test=pcl.has_arg('--test'),
)
if arg.geo is None:
usage('missing --arg')
if arg.geo not in ('census_tract', 'zip5'):
usage('invalid GEO value: ', +arg.geo)
random_seed = 123456
random.seed(random_seed)
path = Path() # use the default dir_input
debug = False
return Bunch(
arg=arg,
debug=debug,
max_sale_price=85e6, # according to Wall Street Journal
path=path,
path_out_csv=path.dir_working() + arg.base_name + '-' + arg.geo +
'.csv',
path_out_occurs=path.dir_working() + arg.base_name + '-' + arg.geo +
'-occurs.pickle',
random_seed=random_seed,
test=arg.test,
)
def make_control(argv):
# return a Bunch
print argv
if not (2 <= len(argv) <= 3):
usage("invalid number of arguments")
pcl = ParseCommandLine(argv)
arg = Bunch(base_name="valrf", yyyymm=argv[1], test=pcl.has_arg("--test"))
try:
arg.yyyymm = int(arg.yyyymm)
except:
usage("YYYYMM not an integer")
random_seed = 123
random.seed(random_seed)
dir_working = Path().dir_working()
debug = False
out_file_name = ("test-" if arg.test else "") + "%s.pickle" % arg.yyyymm
# assure output directory exists
dir_path = dir_working + arg.base_name + "/"
if not os.path.exists(dir_path):
os.makedirs(dir_path)
return Bunch(
arg=arg,
debug=debug,
path_in=dir_working + "samples-train.csv",
path_out=dir_path + out_file_name,
random_seed=random_seed,
test=arg.test,
)
def make_control(argv):
# return a Bunch
print argv
if len(argv) not in (1, 2, 3):
usage('invalid number of arguments')
if len(argv) == 1:
usage()
pcl = ParseCommandLine(argv)
arg = Bunch(
base_name='chart-02',
hp=argv[1],
data=pcl.has_arg('--data'),
test=pcl.has_arg('--test'),
)
random_seed = 123
random.seed(random_seed)
dir_working = Path().dir_working()
debug = False
out_file_name_base = ('test-'
if arg.test else '') + arg.base_name + '-' + arg.hp
return Bunch(
arg=arg,
debug=debug,
path_in_ege=dir_working + 'ege-rfbound-%s-*-folds-10.pickle' % arg.hp,
path_out_base=dir_working + out_file_name_base,
path_data=dir_working + arg.base_name + '-' + arg.hp + '.data.pickle',
random_seed=random_seed,
test=arg.test,
)
def make_control(argv):
# return a Bunch
print argv
if not (1 <= len(argv) <= 2):
usage('invalid number of arguments')
pcl = ParseCommandLine(argv)
arg = Bunch(
base_name='testbest',
test=pcl.has_arg('--test'),
)
random_seed = 123
random.seed(random_seed)
dir_working = Path().dir_working()
debug = False
out_file_name = (('test-' if arg.test else '') + '%s.pickle' % 'results')
# assure output directory exists
dir_path = dir_working + arg.base_name + '/'
if not os.path.exists(dir_path):
os.makedirs(dir_path)
return Bunch(
arg=arg,
debug=debug,
path_in_data=dir_working + 'samples-train.csv',
path_in_best=dir_working + 'chart-06/best.pickle',
path_out=dir_path + out_file_name,
random_seed=random_seed,
test=arg.test,
)
def getFrameInfoFromPath(fitspath):
# Extract frame id from file path
(fitsdir, fitsname) = os.path.split(fitspath)
(frameid, ext) = os.path.splitext(fitsname)
match = re.match('^(\w{3})([AQ])(\d{8})$', frameid)
if match:
(inscode, frametype, frame_no) = match.groups()
frame_no = int(frame_no)
return Bunch(frameid=frameid, fitsname=fitsname,
fitsdir=fitsdir, inscode=inscode,
frametype=frametype, frame_no=frame_no)
raise STARSsimError("path does not match Subaru FITS specification")
def __new__(klass, *args, **kwarg):
self = Bunch.__new__(klass)
#if len(args) > 1:
#raise TypeError('expected at most 1 arguments, got %d' % len(args))
try:
self._root
except:
root = [] # sentinel node
root[:] = [root, root, None]
dict.__setitem__(self, '_root', root)
dict.__setitem__(self, '_map', {})
return self
def __new__(klass,*args,**kwarg):
self = Bunch.__new__(klass)
#if len(args) > 1:
#raise TypeError('expected at most 1 arguments, got %d' % len(args))
try:
self._root
except:
root = [] # sentinel node
root[:] = [root, root, None]
dict.__setitem__(self,'_root',root)
dict.__setitem__(self,'_map' ,{})
return self
def make_control(argv):
'return a Bunch'
print argv
parser = argparse.ArgumentParser()
parser.add_argument('invocation')
parser.add_argument('samples', choices=['all', 'train'])
parser.add_argument('model', choices=['en', 'gb', 'rf'])
parser.add_argument('transaction_month')
parser.add_argument('neighborhood')
parser.add_argument('--test', action='store_true')
parser.add_argument('--trace', action='store_true')
arg = parser.parse_args(argv)
arg.me = arg.invocation.split('.')[0]
if arg.trace:
pdb.set_trace()
# convert arg.neighborhood into arg.all and arg.city
arg.city = (None if arg.neighborhood == 'all' else
arg.neighborhood.replace('_', ' '))
random_seed = 123
random.seed(random_seed)
prior_month = Month(arg.transaction_month).decrement().as_str()
in_dir = '%s-%s-%s-%s' % (arg.samples, arg.model, prior_month,
arg.neighborhood)
out_dir = '%s-%s-%s-%s' % (arg.samples, arg.model, arg.transaction_month,
arg.neighborhood)
dir_working = Path().dir_working()
output_dir = (os.path.join(dir_working, arg.me + '-test', out_dir, '') if
arg.test else os.path.join(dir_working, arg.me, out_dir, ''))
dirutility.assure_exists(output_dir)
return Bunch(
arg=arg,
path_in_fitted=os.path.join(dir_working, 'fit', in_dir, ''),
path_in_samples=os.path.join(dir_working, 'samples2',
arg.samples + '.csv'),
path_out_file=os.path.join(output_dir, 'predictions.pickle'),
path_out_log=os.path.join(output_dir, '0log.txt'),
random_seed=random_seed,
timer=Timer(),
)
def make_control(argv):
'return a Bunch'
print argv
parser = argparse.ArgumentParser()
parser.add_argument('invocation')
parser.add_argument('data', choices=['all', 'train'])
parser.add_argument('model', choices=['en', 'gb', 'rf'])
parser.add_argument('last_month')
parser.add_argument('neighborhood')
parser.add_argument('--test', action='store_true')
parser.add_argument('--trace', action='store_true')
parser.add_argument('--dry', action='store_true') # don't write output
arg = parser.parse_args(argv)
arg.me = arg.invocation.split('.')[0]
if arg.trace:
pdb.set_trace()
arg.last = Month(arg.last_month) # convert to Month and validate value
# convert arg.neighborhood into arg.all and arg.city
arg.city = (
None if arg.neighborhood == 'all' else
arg.neighborhood.replace('_', ' ')
)
random_seed = 123
random.seed(random_seed)
dir_working = Path().dir_working()
fit_dir = (
os.path.join(dir_working, arg.me + '-test') if arg.test else
os.path.join(dir_working, arg.me)
)
last_dir = '%s-%s-%s-%s' % (arg.data, arg.model, arg.last_month, arg.neighborhood)
path_out_dir = os.path.join(fit_dir, last_dir, '')
dirutility.assure_exists(path_out_dir)
return Bunch(
arg=arg,
path_in_dir=os.path.join(dir_working, 'samples2', ''),
path_out_dir=path_out_dir,
path_out_log=os.path.join(path_out_dir, '0log.txt'),
random_seed=random_seed,
timer=Timer(),
)
def make_control(argv):
'return a Bunch'
print argv
parser = argparse.ArgumentParser()
parser.add_argument('training_data',
choices=arg_type.training_data_choices)
parser.add_argument('neighborhood', type=arg_type.neighborhood)
parser.add_argument('model', choices=arg_type.model_choices)
parser.add_argument('n_processes', type=arg_type.n_processes)
parser.add_argument('--cache', action='store_true')
parser.add_argument('--test', action='store_true')
parser.add_argument('--testmapper', action='store_true')
parser.add_argument('--trace', action='store_true')
parser.add_argument('--dry', action='store_true') # don't write output
arg = parser.parse_args(argv[1:])
arg.me = parser.prog.split('.')[0]
if arg.trace:
pdb.set_trace()
random_seed = 123
random.seed(random_seed)
dir_working = Path().dir_working()
dir_final = '%s-%s-%s' % (arg.training_data, arg.neighborhood, arg.model)
if arg.test:
dir_final += '-test'
dir_out = os.path.join(dir_working, arg.me, dir_final)
dirutility.assure_exists(dir_out)
return Bunch(
arg=arg,
path_cache=os.path.join(dir_out, 'cache.pickle'),
path_in_dir_fit_predict=os.path.join(dir_working, 'fit-predict-v2',
''), # TODO: remove v2
path_in_query_samples_all=os.path.join(dir_working, 'samples2',
'all.csv'),
path_in_query_samples_train=os.path.join(dir_working, 'samples2',
'train.csv'),
# path_out_csv=os.path.join(dir_out, 'reduction.csv'),
path_out_dir=dir_out,
# path_out_fitted_attributes=os.path.join(dir_out, 'fitted-attributes.pickle'),
path_out_log=os.path.join(dir_out, '0log.txt'),
random_seed=random_seed,
timer=Timer(),
)
def make_control(argv):
'return a Bunch'
parser = argparse.ArgumentParser()
parser.add_argument('invocation')
parser.add_argument('features_hps_locality', type=arg_type.features_hps_locality)
parser.add_argument('--data', action='store_true')
parser.add_argument('--test', action='store_true')
arg = parser.parse_args(argv)
arg.base_name = 'chart07'
arg.features, arg.hps, arg.locality = arg.features_hps_locality.split('-')
random_seed = 123
random.seed(random_seed)
dir_working = Path().dir_working()
dir_out = dir_working + arg.base_name + '/' + arg.features_hps_locality + '/'
if not os.path.exists(dir_out):
os.makedirs(dir_out)
# fit models for these months
test_months = (
'200512',
'200601', '200602', '200603', '200604', '200605', '200606',
'200607', '200608', '200609', '200610', '200611', '200612',
'200701', '200702', '200703', '200704', '200705', '200706',
'200707', '200708', '200709', '200710', '200711', '200712',
'200801', '200802', '200803', '200804', '200805', '200806',
'200807', '200808', '200809', '200810', '200811', '200812',
'200901', '200902',
)
reduced_file_name = '0data.pickle'
return Bunch(
arg=arg,
debug=False,
k=1, # number of best models examined
path_in_data=dir_out + reduced_file_name,
path_in_valavm_dir=dir_working + ('valavm/%s/' % arg.features_hps_locality),
path_out_data=dir_out + reduced_file_name,
path_out_chart_a_template=dir_out + 'a-nbest-%d-nworst-%d.txt',
path_out_chart_a_pdf=dir_out + 'a-nbest-%d-nworst-%d.pdf',
path_out_chart_b=dir_out + 'b.txt',
path_out_chart_b_pdf=dir_out + 'b.pdf',
test_months=test_months,
timer=Timer(),
)
def make_channels(self, logger):
for ch_i in self.channels:
# Create a log for logging results
if not logger:
queue = Queue.Queue()
log = ssdlog.mklog('ch%d' % ch_i, queue, logging.DEBUG)
else:
log = logger
queue = None
# Create STARS rpc interface object for channel 'ch_i'
iface = STARSchannel(ch_i, self.ev_quit, log, self.taskqueue,
seq_num=self.seq_num)
# Create channel bundle object
self.channel[ch_i] = Bunch(log=log, logqueue=queue, iface=iface)
def build_autoencoder(input, layer_config):
reuse_model = isinstance(layer_config, list)
if not reuse_model:
layer_config
layer_config = layer_config.replace('_', '-').split('-')
layer_config = [parse_input(input)] + [parse(x) for x in layer_config]
if not reuse_model:
ut.print_info('Model config:', color=CONFIG_COLOR)
enc = build_encoder(input, layer_config, reuse=reuse_model)
dec = build_decoder(enc, layer_config, reuse=reuse_model)
mask_list = clean_unpooling_masks(layer_config)
losses = build_losses(layer_config)
return Bunch(
encode=enc,
decode=dec,
losses=losses,
config=layer_config,
mask_list=mask_list)
def make_control(argv):
'return a Bunch'
print argv
parser = argparse.ArgumentParser()
parser.add_argument('invocation')
parser.add_argument('training_data', choices=arg_type.training_data_choices)
parser.add_argument('neighborhood', type=arg_type.neighborhood)
parser.add_argument('model', choices=arg_type.model_choices)
parser.add_argument('prediction_month', type=arg_type.month)
parser.add_argument('--test', action='store_true')
parser.add_argument('--trace', action='store_true')
parser.add_argument('--dry', action='store_true') # don't write output
arg = parser.parse_args(argv)
arg.me = arg.invocation.split('.')[0] + '-v2'
if arg.trace:
pdb.set_trace()
random_seed = 123
random.seed(random_seed)
dir_working = Path().dir_working()
fit_dir = (
os.path.join(dir_working, arg.me + '-test') if arg.test else
os.path.join(dir_working, arg.me)
)
result_dir = '%s-%s-%s-%s' % (arg.training_data, arg.neighborhood, arg.model, arg.prediction_month)
path_out_dir = os.path.join(fit_dir, result_dir, '')
dirutility.assure_exists(path_out_dir)
return Bunch(
arg=arg,
path_in_query_samples=os.path.join(dir_working, 'samples2', 'all.csv'),
path_in_training_samples=os.path.join(dir_working, 'samples2', arg.training_data + '.csv'),
path_out_actuals=os.path.join(path_out_dir, 'actuals.pickle'),
path_out_transaction_ids=os.path.join(path_out_dir, 'transaction_ids.pickle'),
path_out_predictions_attributes=os.path.join(path_out_dir, "predictions-attributes.pickle"),
path_out_dir=path_out_dir,
path_out_feature_names=os.path.join(path_out_dir, 'feature_names.pickle'),
path_out_log=os.path.join(path_out_dir, '0log.txt'),
random_seed=random_seed,
timer=Timer(),
)
def loadPersonality(self, camName, alias=None, moduleDir=None):
# TODO: if reloading, need to call stop() on currently
# executing cam
camName = camName.upper()
# Make sure cam module dir is in our import path
if not moduleDir:
thisDir = os.path.split(sys.modules[__name__].__file__)[0]
moduleDir = '%s/cams/%s' % (thisDir, camName)
if not moduleDir in sys.path:
sys.path.append(moduleDir)
# Instrument will be referred to by this alias
if not alias:
alias = camName
alias = alias.upper()
try:
if self.cams.has_key(alias):
camInfo = self.cams[alias]
self.logger.info("Reloading instrument personality '%s'" % \
camName)
module = camInfo.module
reload(module)
else:
self.logger.debug("Loading instrument personality '%s'" % \
camName)
#moduleName = ("SOSS.cams.%s" % camName)
moduleName = camName
module = __import__(moduleName)
camInfo = Bunch(module=module)
self.cams[alias] = camInfo
except ImportError, e:
self.logger.error("Error loading instrument personality '%s': %s" % \
(camName, str(e)))
self.logger.error("sys.path is '%s'" % ':'.join(sys.path))
self.logger.error("Instrument personality not loaded!")
return
def make_control(argv):
# return a Bunch
print argv
if not (4 <= len(argv) <= 7):
usage('invalid number of arguments')
pcl = ParseCommandLine(argv)
arg = Bunch(
base_name='ege',
folds=pcl.get_arg('--folds'),
rfbound=pcl.get_arg(
'--rfbound'
), # arg.rbound is None or a string or a list of strings
test=pcl.has_arg('--test'),
)
if arg.rfbound is not None:
if not len(arg.rfbound) == 2:
usage('corret is --rfbound HP YYYYMM')
arg.hp = arg.rfbound[0]
arg.yyyymm = arg.rfbound[1]
if not (arg.hp in ('max_depth', 'max_features')):
usage('--rfbound {max_depth|max_features} YYYYMM')
if arg.folds is None:
usage('--folds is required')
else:
arg.folds = int(arg.folds)
random_seed = 123
random.seed(random_seed)
dir_working = Path().dir_working()
debug = False
out_file_name_base = (('test-' if arg.test else '') + arg.base_name +
('' if arg.rfbound is None else '-rfbound-%s-%s' %
(arg.hp, arg.yyyymm)) + ('-folds-%02d' % arg.folds))
return Bunch(
arg=arg,
debug=debug,
path_in=dir_working + 'samples-train-validate.csv',
path_out=dir_working + out_file_name_base + '.pickle',
random_seed=random_seed,
test=arg.test,
)
def make_control(argv):
'return a Bunch'
print argv
parser = argparse.ArgumentParser()
parser.add_argument('filename', type=arg_type.filename_csv)
parser.add_argument('lookback', type=arg_type.positive_int)
parser.add_argument('typical_bid_offer', type=arg_type.positive_int)
parser.add_argument('--test', action='store_true')
parser.add_argument('--trace', action='store_true')
arg = parser.parse_args(argv[1:])
arg.me = parser.prog.split('.')[0]
if arg.trace:
pdb.set_trace()
random_seed = 123
random.seed(random_seed)
dir_working = seven.path.working()
if arg.test:
dir_out = os.path.join(dir_working, arg.me + '-test')
else:
dir_out = os.path.join(dir_working, arg.me)
dirutility.assure_exists(dir_out)
args_str = '%s-%s-%s' % (
arg.filename.split('.')[0],
arg.lookback,
arg.typical_bid_offer,
)
return Bunch(
arg=arg,
path_in_file=os.path.join(seven.path.midpredictor_data(),
arg.filename),
path_out_file_csv=os.path.join(dir_out, args_str + '.csv'),
path_out_file_pickle=os.path.join(dir_out, args_str + '.pickle'),
path_out_log=os.path.join(dir_out, '0log-' + args_str + '.txt'),
random_seed=random_seed,
timer=Timer(),
)
def popitem(self, last=True):
'''od.popitem() -> (k, v), return and remove a (key, value) pair.
Pairs are returned in LIFO order if last is true or FIFO order if false.
'''
if not self:
raise KeyError('dictionary is empty')
root = self._root
if last:
link = root[0]
link_prev = link[0]
link_prev[1] = root
root[0] = link_prev
else:
link = root[1]
link_next = link[1]
root[1] = link_next
link_next[0] = root
key = link[2]
del self._map[key]
value = Bunch.pop(self,key)
return key, value
def popitem(self, last=True):
'''od.popitem() -> (k, v), return and remove a (key, value) pair.
Pairs are returned in LIFO order if last is true or FIFO order if false.
'''
if not self:
raise KeyError('dictionary is empty')
root = self._root
if last:
link = root[0]
link_prev = link[0]
link_prev[1] = root
root[0] = link_prev
else:
link = root[1]
link_next = link[1]
root[1] = link_next
link_next[0] = root
key = link[2]
del self._map[key]
value = Bunch.pop(self, key)
return key, value
def make_control(argv):
# return a Bunch
print argv
parser = argparse.ArgumentParser()
parser.add_argument('invocation')
parser.add_argument('--data', help='reduce input and create data file in WORKING', action='store_true')
parser.add_argument('--test', help='set internal test flag', action='store_true')
arg = Bunch.from_namespace(parser.parse_args(argv))
base_name = arg.invocation.split('.')[0]
random_seed = 123
random.seed(random_seed)
dir_working = Path().dir_working()
debug = False
reduced_file_name = '0data.pickle'
# assure output directory exists
dir_path = dir_working + 'chart01/'
if not os.path.exists(dir_path):
os.makedirs(dir_path)
return Bunch(
arg=arg,
base_name=base_name,
debug=debug,
path_in_samples=dir_working + 'samples-train.csv',
path_out_graph=dir_path + 'median-price.pdf',
path_out_price_volume=dir_path + 'price-volume.pdf',
path_out_stats_all=dir_path + 'price-stats-all.txt',
path_out_stats_2006_2008=dir_path + 'price-stats-2006-2008.txt',
path_reduction=dir_path + reduced_file_name,
random_seed=random_seed,
test=arg.test,
)
