def get_transferParams(host):
key = host.lower()
try:
vals = transferParams[key]
res = Bunch(host=host)
res.__dict__.update(vals)
return res
except KeyError:
pass
# No host-specific parameters. Try the wildcard entry.
try:
vals = transferParams['*']
res = Bunch(host=host)
res.__dict__.update(vals)
return res
except KeyError:
raise STARSsimError("No transfer parameters on file for host '%s'" % \
host)
def make_control(argv):
# return a Bunch
print argv
if not(2 <= len(argv) <= 3):
usage('invalid number of arguments')
pcl = ParseCommandLine(argv)
arg = Bunch(
base_name='valgbr',
yyyymm=argv[1],
test=pcl.has_arg('--test'),
)
try:
arg.yyyymm = int(arg.yyyymm)
except:
usage('YYYYMM not an integer')
random_seed = 123
random.seed(random_seed)
dir_working = Path().dir_working()
debug = False
out_file_name = (
('test-' if arg.test else '') +
'%s.pickle' % arg.yyyymm
)
# assure output directory exists
dir_path = dir_working + arg.base_name + '/'
if not os.path.exists(dir_path):
os.makedirs(dir_path)
fixed_hps = Bunch(
loss='quantile',
alpha=0.5,
n_estimators=1000,
max_depth=3,
max_features=None)
return Bunch(
arg=arg,
debug=debug,
fixed_hps=fixed_hps,
path_in=dir_working + 'samples-train.csv',
path_out=dir_path + out_file_name,
random_seed=random_seed,
test=arg.test,
)
def make_control(argv):
# return a Bunch
print argv
if not(4 <= len(argv) <= 5):
usage('invalid number of arguments')
pcl = ParseCommandLine(argv)
arg = Bunch(
base_name='rfbound',
hp=argv[1],
yyyymm=argv[2],
folds=argv[3],
test=pcl.has_arg('--test'),
)
try:
arg.folds = int(arg.folds)
except:
usage('INT not an integer; ' + str(arg.folds))
random_seed = 123
random.seed(random_seed)
dir_working = Path().dir_working()
debug = False
out_file_name = (
'%s/%s%s-%s-folds-%02d.pickle' % (
arg.base_name,
('test-' if arg.test else ''),
arg.hp,
arg.yyyymm,
arg.folds)
)
# assure the output directory exists
dir_path = dir_working + arg.base_name
if not os.path.exists(dir_path):
os.makedirs(dir_path)
return Bunch(
arg=arg,
debug=debug,
path_in=dir_working + 'samples-train-validate.csv',
path_out=dir_working + out_file_name,
random_seed=random_seed,
test=arg.test,
)
def make_control(argv):
# return a Bunch
print argv
if not (4 <= len(argv) <= 7):
usage('invalid number of arguments')
pcl = ParseCommandLine(argv)
arg = Bunch(
base_name='ege',
folds=pcl.get_arg('--folds'),
rfbound=pcl.get_arg(
'--rfbound'
), # arg.rbound is None or a string or a list of strings
test=pcl.has_arg('--test'),
)
if arg.rfbound is not None:
if not len(arg.rfbound) == 2:
usage('corret is --rfbound HP YYYYMM')
arg.hp = arg.rfbound[0]
arg.yyyymm = arg.rfbound[1]
if not (arg.hp in ('max_depth', 'max_features')):
usage('--rfbound {max_depth|max_features} YYYYMM')
if arg.folds is None:
usage('--folds is required')
else:
arg.folds = int(arg.folds)
random_seed = 123
random.seed(random_seed)
dir_working = Path().dir_working()
debug = False
out_file_name_base = (('test-' if arg.test else '') + arg.base_name +
('' if arg.rfbound is None else '-rfbound-%s-%s' %
(arg.hp, arg.yyyymm)) + ('-folds-%02d' % arg.folds))
return Bunch(
arg=arg,
debug=debug,
path_in=dir_working + 'samples-train-validate.csv',
path_out=dir_working + out_file_name_base + '.pickle',
random_seed=random_seed,
test=arg.test,
)
def test_Constructor_ShufflesProperly_TwoBunchesAreNotTheSame(self) :
# Arrange & Act
tiles = []
tileA = Tile('A', 1, 1, 1)
tileB = Tile('B', 2, 2, 3)
tileC = Tile('C', 3, 3, 5)
tiles.append(tileA)
tiles.append(tileB)
tiles.append(tileC)
bunchOne = Bunch(tiles)
bunchTwo = Bunch(tiles)
# Assert
self.assertNotEqual(bunchOne, bunchTwo)
def make_control(argv):
'return a Bunch'
print argv
parser = argparse.ArgumentParser()
parser.add_argument('invocation')
parser.add_argument('--test', action='store_true')
parser.add_argument('--trace', action='store_true')
arg = parser.parse_args(argv)
arg.me = arg.invocation.split('.')[0]
if arg.trace:
pdb.set_trace()
random_seed = 123
random.seed(random_seed)
dir_working = Path().dir_working()
dir_out = os.path.join(dir_working, arg.me + ('-test' if arg.test else ''))
dirutility.assure_exists(dir_out)
return Bunch(
arg=arg,
path_in_samples=os.path.join(dir_working, 'samples2', 'train.csv'),
# path_out_csv=os.path.join(dir_out, 'reduction.csv'),
path_out_report_by_price=os.path.join(dir_out, 'report-by-price.txt'),
path_out_report_by_n_trades=os.path.join(dir_out,
'report-by-n-trades.txt'),
path_out_log=os.path.join(dir_out, '0log.txt'),
random_seed=random_seed,
timer=Timer(),
)
def get_stats_template():
return Bunch(batch=[],
input=[],
encoding=[],
reconstruction=[],
total_loss=0.,
start=time.time())
def make_grid():
'return all hyperparameter grid points'
return Bunch(
# HP settings to test across all models
n_months_back=(1, 2, 3, 6, 12),
# HP settings to test for ElasticNet models
alpha=(0.01, 0.03, 0.1, 0.3, 1.0), # multiplies the penalty term
l1_ratio=(0.0, 0.25, 0.50, 0.75,
1.0), # 0 ==> L2 penalty, 1 ==> L1 penalty
units_X=('natural', 'log'),
units_y=('natural', 'log'),
# HP settings to test for tree-based models
# settings based on Anil Kocak's recommendations
n_estimators=(10, 30, 100, 300),
max_features=(1, 'log2', 'sqrt', 'auto'),
max_depth=(1, 3, 10, 30, 100, 300),
# HP setting to test for GradientBoostingRegression models
learning_rate=(.10, .25, .50, .75, .99),
# experiments demonstrated that the best loss is seldom quantile
# loss_seq=('ls', 'quantile'),
loss=('ls', ),
)
def __init__(self, phytomerNumber, rank, state, treeInstance, day, month,
year, TT, initiationday, initiationmonth, initiationyear,
initiationTT):
#self.name = "Phytomer_" + repr(phytomerNumber)
self.name = phytomerNumber
self.rank = rank
self.state = state
self.tree = treeInstance
self.appearanceDay = day
self.appearanceMonth = month
self.appearanceYear = year
self.appearanceTT = TT
self.initiationDay = initiationday
self.initiationMonth = initiationmonth
self.initiationYear = initiationyear
self.initiationTT = initiationTT
self.deathday = 0
self.deathmonth = 0
self.deathyear = 0
self.deathTT = 0
self.leaf = Leaf(self)
self.bunch = Bunch(self)
self.internode = Internode(self)
self.demand = 0
self.step_apparition = self.tree.simulation.step
def get_displays(loc):
try:
b1 = locations[loc.lower()]
num_displays = len(b1.hosts.keys())
res = []
for i in xrange(num_displays):
b2 = b1.hosts['h%d' % (i+1)]
up = b2.displays.d1
dn = b2.displays.d2
res.append((Bunch(host=b2.host, xdisp=up.xdisp, geom=up.geometry),
Bunch(host=b2.host, xdisp=dn.xdisp, geom=dn.geometry))
)
except KeyError, e:
raise e
def get_hosts(insname):
try:
# Query the name server on the Gen2 host for the service
# names of the instrument and the status subsystems
ro.init()
ns = ro.getns()
insint_hosts = ns.getHosts(insname)
if len(insint_hosts) == 0:
raise Error("No instrument interface found")
status_hosts = ns.getHosts('status')
if len(insint_hosts) == 0:
raise Error("No status interface found")
# Strip off FQDN to short name
cmds = insint_hosts[0][0].split('.')[0]
sdst = status_hosts[0][0].split('.')[0]
d = Bunch(obshost=cmds, gethost=cmds, obchost=cmds,
stathost=sdst)
return d
except Exception, e:
raise Error("Couldn't get information: %s" % str(e))
def make_control(argv):
# return a Bunch
print argv
if len(argv) not in (1, 2):
usage('invalid number of arguments')
parser = argparse.ArgumentParser()
parser.add_argument('invocation')
parser.add_argument('--test', action='store_true')
arg = parser.parse_args(argv)
arg.base_name = arg.invocation.split('.')[0]
path = Path()
dir_out = dirutility.assure_exists(path._dir_working + arg.base_name +
('-test' if arg.test else '')) + '/'
file_out = path._dir_working + arg.base_name + '-derived' + (
'-test' if arg.test else '') + '.csv'
path_in = path._dir_input + 'neighborhood-data/census.csv'
random_seed = 123456
random.seed(random_seed)
debug = False
return Bunch(
arg=arg,
debug=debug,
path_in=path_in,
path_out=file_out,
path_out_log=dir_out + '0log.txt',
random_seed=random_seed,
test=arg.test,
)
def calc_cron(hr='*', min='*', mon='*', day='*', wday='*'):
cron = Bunch(hr=hr, min=min, mon=mon, day=day, wday=wday)
def docalc(bm, svcinfo):
# Get current time
(yr, mon, day, hr, min, sec, wday, yday,
isdst) = time.localtime(time.time())
if cron.mon != '*':
if not (mon in cron.mon):
return
if cron.day != '*':
if not (day in cron.day):
return
if cron.wday != '*':
if not (wday in cron.wday):
return
if cron.hr != '*':
if not (hr in cron.hr):
return
if cron.min != '*':
if not (min in cron.min):
return
# Don't run the same cron job if less than 60 seconds have expired
cur_time = time.time()
if svcinfo.has_key('time_last') and (cur_time - svcinfo['time_last'] <
60.0):
return
svcinfo['time_last'] = cur_time
svcinfo['time_next'] = cur_time
return docalc
def make_control(argv):
'return a Bunch'
print argv
parser = argparse.ArgumentParser()
parser.add_argument('--test', action='store_true')
parser.add_argument('--trace', action='store_true')
arg = parser.parse_args(argv[1:])
arg.me = parser.prog.split('.')[0]
if arg.trace:
pdb.set_trace()
random_seed = 123
random.seed(random_seed)
dir_working = Path().dir_working()
dir_out = os.path.join(dir_working, arg.me)
dirutility.assure_exists(dir_out)
return Bunch(
arg=arg,
path_in_dir=os.path.join(dir_working, 'fit-predict-v2'),
path_out_no_data=os.path.join(dir_out, 'no_data.pickle'),
path_out_reduction=os.path.join(dir_out, 'reduction.pickle'),
path_out_reduction_2007=os.path.join(dir_out, 'reduction_2007.pickle'),
path_out_reduction_200701=os.path.join(dir_out, 'reduction_200701.pickle'),
path_out_log=os.path.join(dir_out, '0log.txt'),
random_seed=random_seed,
timer=Timer(),
)
def lookup_rpcsvc(key):
"""Get the send and receive program numbers for a service
looked up by a service string. May raise a KeyError."""
recv, send = ProgramNumbers[key]
return Bunch(server_send_prgnum=send,
server_receive_prgnum=recv,
client_send_prgnum=recv,
client_receive_prgnum=send)
def make_control(argv):
# return a Bunch
print argv
if not(2 <= len(argv) <= 3):
usage('invalid number of arguments')
pcl = ParseCommandLine(argv)
arg = Bunch(
base_name='linval',
yyyymm=argv[1],
test=pcl.has_arg('--test'),
)
try:
arg.yyyymm = int(arg.yyyymm)
except:
usage('YYYYMM not an integer')
random_seed = 123
random.seed(random_seed)
dir_working = Path().dir_working()
debug = False
out_file_name = (
('test-' if arg.test else '') +
'%s.pickle' % arg.yyyymm
)
# assure output directory exists
dir_path = dir_working + arg.base_name + '/'
if not os.path.exists(dir_path):
os.makedirs(dir_path)
return Bunch(
arg=arg,
debug=debug,
path_in=dir_working + 'samples-train-validate.csv',
path_out=dir_path + out_file_name,
random_seed=random_seed,
test=arg.test,
)
def make_control(argv):
# return a Bunch
print argv
if len(argv) not in (1, 2, 3):
usage('invalid number of arguments')
pcl = ParseCommandLine(argv)
arg = Bunch(
base_name='chart-05',
data=pcl.has_arg('--data'),
test=pcl.has_arg('--test'),
)
if len(argv) == 3:
both = arg.data and arg.test
if not both:
usage('there is an extra invocation option')
random_seed = 123
random.seed(random_seed)
dir_working = Path().dir_working()
debug = False
reduced_file_name = ('test-' if arg.test else '') + 'data.pickle'
# assure output directory exists
dir_path = dir_working + arg.base_name + '/'
if not os.path.exists(dir_path):
os.makedirs(dir_path)
return Bunch(
arg=arg,
debug=debug,
path_in_ege=dir_working + 'valgbr/*.pickle',
path_reduction=dir_path + reduced_file_name,
path_chart_base=dir_path,
random_seed=random_seed,
test=arg.test,
)
def get_floaters(hostbnches):
"""Map floating interface names to generic node names."""
res = Bunch()
for key in hostbnches.keys():
try:
alt_ifcs = hostbnches[key]['alt']
for name in alt_ifcs:
res[name] = key
except KeyError:
continue
return res
def test_isBunchEmpty_BunchIsEmpty_ReturnsTrue(self) :
# Arrange
tiles = []
bunch = Bunch(tiles)
# Act
bunch.DealFromBunch(144)
result = bunch.IsBunchEmpty()
# Assert
self.assertTrue(result)
def test_Peel_ValidBunch_ReturnsTilesFromBunch(self) :
# Arrange
tiles = []
tile = Tile('A', 1, 1, 1)
tiles.append(tile)
bunch = Bunch(tiles)
# Act
result = bunch.Peel()
# Assert
self.assertTrue(result)
def __init__(self, heuristic):
self.bunch = Bunch()
handTiles = self.bunch.DealFromBunch(15)
self.hand = Hand("BRI", handTiles)
self.bunch.DealFromBunch(15)
self.board = Board()
self.board.PrintBoard()
self.concurrentExceptions = 0
self.bri = BRI(heuristic)
self.time = 1000
self.timer = self.time #nanoseconds
def make_control(argv):
# return a Bunch
print argv
if len(argv) not in (1, 2):
usage('invalid number of arguments')
pcl = ParseCommandLine(argv)
if pcl.has_arg('--help'):
usage()
arg = Bunch(
base_name=argv[0].split('.')[0],
test=pcl.has_arg('--test'),
)
random_seed = 123456
random.seed(random_seed)
path = Path() # use the default dir_input
debug = False
file_out_transactions = (('testing-' if arg.test else '') + arg.base_name +
'-al-g-sfr' + '.csv')
return Bunch(
arg=arg,
debug=debug,
max_sale_price=85e6, # according to Wall Street Journal
path=path,
path_in_census_features=path.dir_working() +
'census-features-derived.csv',
path_in_parcels_features_census_tract=path.dir_working() +
'parcels-features-census_tract.csv',
path_in_parcels_features_zip5=path.dir_working() +
'parcels-features-zip5.csv',
path_out_transactions=path.dir_working() + file_out_transactions,
random_seed=random_seed,
test=arg.test,
)
def __init__(self, timestep, totaltime, intmethod, ElectricF, MagneticF,
BunchSize):
self.timestep = timestep
self.totaltime = totaltime
self.intmethod = intmethod
self.ElectricF = ElectricF
self.MagneticF = MagneticF
self.BunchSize = BunchSize
Field = EMField(Magnetic=self.MagneticF, Electric=self.ElectricF)
self.Field = Field
ParticleBunch = Bunch(self.BunchSize)
self.Bunch = ParticleBunch
self.Bunch.CreateBunch()
def _on_training_start(self, sess):
# Writers and savers
self.summary_writer = tf.summary.FileWriter(FLAGS.logdir, sess.graph)
self.saver = tf.train.Saver()
self._build_embedding_saver(sess)
self._restore_model(sess)
# Loss summaries
self._build_summaries()
self.epoch_stats = get_stats_template()
self.stats = Bunch(epoch_accuracy=[],
epoch_reconstructions=[],
permutation=None)
def test_Constructor_GetBunch_ExpectedTilesPresentInBunch(self) :
# Arrange
tiles = []
tile = Tile('A', 1, 1, 1)
tiles.append(tile)
expected = 1
bunch = Bunch(tiles)
# Act
result = len(bunch.GetBunch())
# Assert
self.assertEqual(result, expected)
def test_ScoreBunch_ScoreOfOne_ScoreOfOne(self) :
# Arrange
tiles = []
tile = Tile('A', 1, 1, 1)
tiles.append(tile)
expected = 1
bunch = Bunch(tiles)
# Act
result = bunch.ScoreBunch()
# Assert
self.assertEqual(result, expected)
def convert_csv_to_yolo_format(in_dir, out_dir, gt_csv_file_path):
'''This Fn takes the input csv file and write the images and texts file as in yolo format.'''
counter = 0
total_roi_instances = 0
with open(gt_csv_file_path, 'r') as csvfile:
samples = csv.reader(csvfile, delimiter=',')
for sample in samples:
print('row count ', counter)
counter += 1
img_name = str(sample[0])
img_path = os.path.join(in_dir, img_name)
img_no = get_file_number(img_name)
img = cv2.imread(img_path)
no_instances = int(sample[1])
total_roi_instances += no_instances
lstIndex = 2
list_instances = []
for ii in range(no_instances):
x_min = float(sample[lstIndex])
lstIndex += 1
y_min = float(sample[lstIndex])
lstIndex += 1
x_max = float(sample[lstIndex])
lstIndex += 1
y_max = float(sample[lstIndex])
lstIndex += 1
# class_lbl = list_class_name.index(class_name)
class_lbl = int(sample[lstIndex])
lstIndex += 1
instance = Bunch(x_min=x_min,
y_min=y_min,
x_max=x_max,
y_max=y_max,
lbl=class_lbl)
list_instances.append(instance)
llist_yolo = bunch_2_yolo.get_yolo_llist(list_instances, img)
txt_path, txt_name = image_utils.get_txt_path_name_frm_index(
int(img_no), out_dir)
old_img_path = os.path.join(in_dir, img_name)
new_img_path, new_img_name = image_utils.get_img_path_name_frm_index(
int(img_no), out_dir)
copyfile(old_img_path, new_img_path)
f = open(txt_path, 'w')
writer = csv.writer(f, delimiter=' ')
for list_yolo in llist_yolo:
writer.writerow(list_yolo)
f.close()
print('total roi instances = ', total_roi_instances)
def make_control(argv):
# return a Bunch
print argv
if len(argv) not in (1, 2):
usage('invalid number of arguments')
pcl = ParseCommandLine(argv)
arg = Bunch(
base_name='chart-07',
data=pcl.has_arg('--data'),
test=pcl.has_arg('--test'),
)
random_seed = 123
random.seed(random_seed)
dir_working = Path().dir_working()
debug = False
reduced_file_name = ('test-' if arg.test else '') + 'data.pickle'
# assure output directory exists
dir_path = dir_working + arg.base_name + '/'
if not os.path.exists(dir_path):
os.makedirs(dir_path)
return Bunch(
arg=arg,
debug=debug,
path_in_best=dir_working + 'chart-06/best.pickle',
path_in_samples='../data/working/samples-train.csv',
path_reduction=dir_path + reduced_file_name,
path_chart_base=dir_path,
random_seed=random_seed,
test=arg.test,
)
def make_control(argv):
# return a Bunch
print argv
if len(argv) not in (1, 2):
usage('invalid number of arguments')
pcl = ParseCommandLine(argv)
arg = Bunch(
base_name=argv[0].split('.')[0],
test=pcl.has_arg('--test'),
)
random_seed = 123
random.seed(random_seed)
dir_working = Path().dir_working()
debug = False
out_file_name_base = ('testing-' if arg.test else '') + arg.base_name
return Bunch(
arg=arg,
debug=debug,
fraction_test=0.2,
max_sale_price=85e6, # according to Wall Street Journal
path_in=dir_working + 'transactions-al-g-sfr.csv',
path_out_info_reasonable=dir_working + out_file_name_base +
'-info-reasonable.pickle',
path_out_test=dir_working + out_file_name_base + '-test.csv',
path_out_train=dir_working + out_file_name_base + '-train.csv',
path_out_train_validate=dir_working + out_file_name_base +
'-train-validate.csv',
path_out_validate=dir_working + out_file_name_base + '-validate.csv',
random_seed=random_seed,
test=arg.test,
)
def test_IsBunchEmpty_BunchNotEmpty_ReturnsFalse(self) :
# Arrange
tiles = []
tileA = Tile('A', 1, 1, 1)
tileB = Tile('B', 2, 2, 3)
tiles.append(tileA)
tiles.append(tileB)
bunch = Bunch(tiles)
# Act
result = bunch.IsBunchEmpty()
# Assert
self.assertFalse(result)
