def get_presence(self, port_num):
# Check for invalid port_num
if port_num < self.first_port or port_num > self.last_port:
return False
cpld_index = (port_num / CPLD_PORT_NUM) + 1
index = (port_num % CPLD_PORT_NUM) + 1
if cpld_index == 5:
path = self.sfpplus_present_path
port_path = path.format(0)
else:
path = self.present_path
port_path = path.format(self.port_to_i2cbus_mapping[cpld_index], index)
try:
reg_file = open(port_path, 'r')
reg_value = int(reg_file.readline().rstrip(), 16)
reg_file.close()
except IOError as e:
print "Error: unable to open file: %s" % str(e)
return False
if cpld_index == 5:
if reg_value == 3:
# SFP+ 1 and 0 is not present, 0 is present
reg_value = 0
else:
if index == 1:
reg_value = (reg_value < 2)
else:
reg_value = ((reg_value % 2) == 0)
return reg_value
def get_presence(self):
"""
Retrieves the presence of the device
Returns:
bool: True if device is present, False if not
"""
cpld_index = (self.index / CPLD_PORT_NUM) + 1
index = (self.index % CPLD_PORT_NUM) + 1
if cpld_index == 5:
path = self.sfpplus_present_path
port_path = path.format(0)
else:
path = self.present_path
port_path = path.format(self.port_to_i2cbus_mapping[cpld_index],
index)
try:
reg_file = open(port_path, 'r')
reg_value = int(reg_file.readline().rstrip(), 16)
reg_file.close()
except IOError as e:
print "Error: unable to open file: %s" % str(e)
return False
if cpld_index == 5:
if reg_value == 3:
# SFP+ 1 and 0 is not present, 0 is present
reg_value = 0
else:
if index == 1:
reg_value = (reg_value < 2)
else:
reg_value = ((reg_value % 2) == 0)
return reg_value
def load_models(path, cam_id_list):
"""Load the pickled dictionary of model from disk
and fill the model dictionary.
Parameters
----------
path : string
The path where the pre-trained, pickled models are
stored. `path` is assumed to contain a `{cam_id}` keyword
to be replaced by each camera identifier in `cam_id_list`
(or at least a naked `{}`).
cam_id_list : list
List of camera identifiers like telescope ID or camera ID
and the assumed distinguishing feature in the filenames of
the various pickled regressors.
Returns
-------
model_dict: dict
Dictionary with `cam_id` as keys and pickled models as values.
"""
model_dict = {}
for key in cam_id_list:
try:
model_dict[key] = joblib.load(path.format(cam_id=key))
except IndexError:
model_dict[key] = joblib.load(path.format(key))
return model_dict
def __init__(self, image_path, save_images=True):
self.image_path = image_path
self.image_name = os.path.basename(self.image_path)
# Convert image to grayscale
self.image_array_grayscale = cv2.imread(image_path,
cv2.IMREAD_GRAYSCALE)
self.image_lbp = ImageDescriptor.get_lbp_image(
self.image_array_grayscale)
self.gradient_x, self.gradient_y = ImageDescriptor.image_gradient(
self.image_lbp)
self.energy = numpy.abs(self.gradient_x) + numpy.abs(self.gradient_y)
self.wiener = scipy.signal.wiener(self.image_lbp, 5)
self.noise = self.image_lbp - self.wiener
self.vertical_cumulative_energy_transposed = self.minimum_cumulative_energy(
self.energy.transpose())
self.horizontal_cumulative_energy = self.minimum_cumulative_energy(
self.energy)
if save_images:
path = os.path.join('.', 'processed', 'yin',
'{}_' + self.image_name)
cv2.imwrite(path.format('lbp'), self.image_lbp)
cv2.imwrite(path.format('gx'), self.gradient_x)
cv2.imwrite(path.format('gy'), self.gradient_y)
cv2.imwrite(path.format('wiener'), self.wiener)
cv2.imwrite(path.format('noise'), self.noise)
def sendEmail(self, country, totPopulation, variation):
path = Configuration().getProperty('DEFAULT', 'emailaddressespath')
#get email template
emailtexttemplatepath = path.format(Configuration().getProperty(
'DEFAULT', 'emailtext'))
with open(emailtexttemplatepath, 'r') as template_file:
template_file_content = template_file.read()
#get country addresses
countryaddressespath = path.format(Configuration().getProperty(
'DEFAULT', 'countryemailfilename').format(country))
if os.path.exists(countryaddressespath):
countryaddresses = pd.read_json(countryaddressespath)
else:
countryaddresses = None
#get admin addresses
adminaddressespath = path.format(Configuration().getProperty(
'DEFAULT', 'adminemailfilename'))
if os.path.exists(adminaddressespath):
adminaddresses = pd.read_json(adminaddressespath)
else:
adminaddresses = None
msg = MIMEMultipart() # create a message
# add in the actual data to the message template
message = template_file_content.format(country_id=country,
totpop=totPopulation,
variation=variation)
msg['Subject'] = "Food security checker variation"
#sender address
msg['From'] = Configuration().getProperty('DEFAULT',
'emailsenderaddress')
#add country addresses
if countryaddresses is not None:
for contact in countryaddresses["addresses"]:
if contact["role"] == "to":
msg['To'] = contact["email"]
if contact["role"] == "cc":
msg['Cc'] = contact["email"]
if contact["role"] == "ccn":
msg['Ccn'] = contact["email"]
#add admin addresses
if adminaddresses is not None:
for contact in adminaddresses["addresses"]:
if contact["role"] == "to":
msg['To'] = contact["email"]
if contact["role"] == "cc":
msg['Cc'] = contact["email"]
if contact["role"] == "ccn":
msg['Ccn'] = contact["email"]
msg.attach(MIMEText(message, 'plain'))
def create(version, category):
# type: (str) -> None
""" Create a version file with default values. """
# Compute installers checksum
checksum_dmg = checksum_exe = checksum_exe_admin = None
paths = (
"dist/nuxeo-drive-{}.dmg",
"dist/nuxeo-drive-{}.exe",
"dist/nuxeo-drive-{}-admin.exe",
)
for path in paths:
if os.getenv("TESTING") and not os.path.isfile(path.format(version)):
continue
with open(
path.format(version), "rb" # Set TESTING=1 envar to skip the error
) as installer:
checksum = hashlib.sha256(installer.read()).hexdigest()
if path.endswith("dmg"):
checksum_dmg = checksum
elif "admin" in path:
checksum_exe_admin = checksum
else:
checksum_exe = checksum
# Create the version file
output = "{}.yml".format(version)
"""
We set 10.3-SNAPSHOT to allow presales to test the current dev version.
Same for the future 10.10 to not block updates when it will be available.
Note that we removed the following section with NXDRIVE-1419:
min_all:
"7.10": "7.10-HF47"
"8.10": "8.10-HF38"
"9.10": "9.10-HF20"
"10.3": "10.3-SNAPSHOT"
"10.10": "10.10-SNAPSHOT"
"""
yml = """{}:
min: "7.10"
type: {}
checksum:
algo: sha256
dmg: {}
exe: {}
exe-admin: {}
""".format(
version, category, checksum_dmg, checksum_exe, checksum_exe_admin
)
with open(output, "w") as versions:
versions.write(yml)
def parse_matlab_basis_sets(path):
""" Load matlab generated basis sets files,
The expected format for the `path` argument is a string of the form
"some_basis_set_{}_1.bsc" where "{}" will be replaced by "" for
the first file and "pr" for the second. Gzip compressed text files
are accepted. For instance:
basis1000_1.bst.gz
basis1000pr_1.bst.gz
"""
M = np.loadtxt(path.format('pr'))
Mc = np.loadtxt(path.format(''))
return M, Mc
def __init__(self):
path = self.present_path
port_path = path.format(self.port_to_i2cbus_0[1], 1)
if not os.path.exists(port_path):
self.port_to_i2cbus_mapping = self.port_to_i2cbus_1
else:
self.port_to_i2cbus_mapping = self.port_to_i2cbus_0
path = self.eeprom_path
for x in range(self.first_port, self.last_port + 1):
index = (x % 8)
i2c_index = (x / 8) + 1
self.port_to_eeprom[x] = path.format(
self.port_to_i2cbus_mapping[i2c_index], (index + 1))
SfpUtilBase.__init__(self)
def reset(self, port_num):
# Check for invalid port_num
if port_num < self.first_port or port_num > self.last_port:
return False
cpld_index = (port_num / CPLD_PORT_NUM) + 1
index = (port_num % CPLD_PORT_NUM) + 1
if cpld_index == 5:
# Not support reset for SFP+
return True
else:
path = self.port_reset_path
port_path = path.format(self.port_to_i2cbus_mapping[cpld_index], index)
try:
reg_file = open(port_path, 'w')
except IOError as e:
print "Error: unable to open file: %s" % str(e)
return False
# reset
reg_file.write('1')
time.sleep(1)
reg_file.write('0')
reg_file.close()
return True
def get_presence_all(self):
port = self.port_start
bitmap = ""
while (port >= self.port_start) and (port <= self.port_end):
index = (port % 8)
i2c_index = (port / 8) + 1
path = self.present_path
port_path = path.format(self.port_to_i2cbus_mapping[i2c_index],
(index + 1))
try:
reg_file = open(port_path)
except IOError as e:
print "Error: unable to open file: %s" % str(e)
return False
reg_value = reg_file.readline().rstrip()
reg_file.close()
if reg_value == '1':
bitmap += '1' + " "
else:
bitmap += '0' + " "
port += 1
content = bitmap.split(" ")
content = "".join(content[::-1])
return int(content, 2)
def download(info,
url = 'http://www.nadaguides.com/Cars/{year}/{make}/{model}/{trim}/Values',
file_fmt = '{year}-{make}-{model}-{trim}.html',
years = range(2000, 2016),
path = './html/{make}-{model}-{trim}/'):
"""info should be a dict with make, model, and trim members
"""
path = path.format(**info)
if not os.path.exists(path):
os.makedirs(path)
for year in years:
info['year'] = year
dlurl = url.format(**info)
filename = os.path.join(path, file_fmt.format(**info))
logging.debug("downloading {} --> {}".format(dlurl, filename))
if os.path.exists(filename):
logging.warn(" SKIP -- output file already exists.")
return
r = requests.get(dlurl)
if r.status_code == requests.status_codes.codes.ok:
with open (filename, 'w') as f:
f.write(r.text)
else:
logging.warn(" status -- {}".format(r.status_code))
def set_lpmode(self, lpmode):
"""
Sets the lpmode (low power mode) of SFP
Args:
lpmode: A Boolean, True to enable lpmode, False to disable it
Note : lpmode can be overridden by set_power_override
Returns:
A boolean, True if lpmode is set successfully, False if not
"""
index = (self.index % 8)
i2c_index = (self.index / 8) + 1
path = self.lpmode_path
if i2c_index == 5:
raise NotImplementedError
else:
port_path = path.format(self.port_to_i2cbus_mapping[i2c_index],
(index + 1))
try:
reg_file = open(port_path, 'w')
except IOError as e:
print "Error: unable to open file: %s" % str(e)
return False
if lpmode == True:
reg_file.write('1')
else:
reg_file.write('0')
reg_file.close()
return True
def get_lpmode(self):
"""
Retrieves the lpmode (low power mode) status of this SFP
Returns:
A Boolean, True if lpmode is enabled, False if disabled
"""
index = (self.index % 8)
i2c_index = (self.index / 8) + 1
path = self.lpmode_path
if i2c_index == 5:
return False
else:
port_path = path.format(self.port_to_i2cbus_mapping[i2c_index],
(index + 1))
try:
reg_file = open(port_path)
reg_value = int(reg_file.readline().rstrip(), 16)
reg_file.close()
except IOError as e:
print "Error: unable to open file: %s" % str(e)
return False
return reg_value
def get_reset_status(self):
"""
Retrieves the reset status of SFP
Returns:
A Boolean, True if reset enabled, False if disabled
"""
cpld_index = (self.index / CPLD_PORT_NUM) + 1
index = (self.index % CPLD_PORT_NUM) + 1
if cpld_index == 5:
# Not support reset for SFP+
return True
path = self.port_reset_path
port_path = path.format(self.port_to_i2cbus_mapping[cpld_index], index)
try:
reg_file = open(port_path)
reg_value = int(reg_file.readline().rstrip(), 16)
reg_file.close()
except IOError as e:
print "Error: unable to open file: %s" % str(e)
return False
return reg_value
def reset(self, port_num):
# Check for invalid port_num
if port_num < self.first_port or port_num > self.last_port:
return False
index = (port_num % 8)
i2c_index = (port_num / 8) + 1
path = self.port_reset_path
port_path = path.format(self.port_to_i2cbus_mapping[i2c_index],
(index + 1))
try:
reg_file = open(port_path, 'w')
except IOError as e:
print "Error: unable to open file: %s" % str(e)
return False
# reset
reg_file.write('1')
time.sleep(1)
reg_file.write('0')
reg_file.close()
return True
def get_presence(self, port_num):
# Check for invalid port_num
if port_num < self.port_start or port_num > self.port_end:
return False
index = (port_num % 8)
i2c_index = (port_num / 8) + 1
path = self.present_path
port_path = path.format(self.port_to_i2cbus_mapping[i2c_index],
(index + 1))
try:
reg_file = open(port_path)
except IOError as e:
print "Error: unable to open file: %s" % str(e)
return False
reg_value = reg_file.readline().rstrip()
reg_file.close()
if reg_value == '1':
return True
return False
def load_image(data_dir, filename):
image = utils.load_image(data_dir, filename, n_rows, n_cols)
logits = np.array(image_to_logits(image))
max = np.amax(logits)
scale = 255.0 / max
logits *= scale
# save it
path = '/home/dev/code/nn/robotcar/movement/ldata/{0}'
cv2.imwrite(path.format(filename), logits)
#pdb.set_trace()
#si(logits)
'''
show_logits = np.array(logits)
for r in range(show_logits.shape[0]):
for c in range(show_logits.shape[1]):
if logits[r][c] > 0.0:
show_logits[r][c] = 255*logits[r][c]
# 'bsfrf_20180605061616-1.jpg'
si(image_to_target(image), "target")
si(image, "image")
si(show_logits, "logits")
pdb.set_trace()
'''
return logits
def set_low_power_mode(self, port_num, lpmode):
# Check for invalid port_num
if port_num < self.first_port or port_num > self.last_port:
return False
index = (port_num % 8)
i2c_index = (port_num / 8) + 1
path = self.lpmode_path
if i2c_index == 5:
raise NotImplementedError
else:
port_path = path.format(self.port_to_i2cbus_mapping[i2c_index], (index + 1))
try:
reg_file = open(port_path, 'w')
except IOError as e:
print "Error: unable to open file: %s" % str(e)
return False
if lpmode == True:
reg_file.write('1')
else:
reg_file.write('0')
reg_file.close()
return True
def get_bounding_box(self):
"""Get the bounding box of this file."""
from pyproj import Geod
geod = Geod(ellps='WGS84')
dataset_group = DATASET_KEYS[self.datasets[0]]
idx = 0
lons_ring = None
lats_ring = None
while True:
path = 'Data_Products/{dataset_group}/{dataset_group}_Gran_{idx}/attr/'
prefix = path.format(dataset_group=dataset_group, idx=idx)
try:
lats = self.file_content[prefix + 'G-Ring_Latitude']
lons = self.file_content[prefix + 'G-Ring_Longitude']
if lons_ring is None:
lons_ring = lons
lats_ring = lats
else:
prev_lon = lons_ring[0]
prev_lat = lats_ring[0]
dists = list(geod.inv(lon, lat, prev_lon, prev_lat)[2] for lon, lat in zip(lons, lats))
first_idx = np.argmin(dists)
if first_idx == 2 and len(lons) == 8:
lons_ring = np.hstack((lons[:3], lons_ring[:-2], lons[4:]))
lats_ring = np.hstack((lats[:3], lats_ring[:-2], lats[4:]))
else:
raise NotImplementedError("Don't know how to handle G-Rings of length %d" % len(lons))
except KeyError:
break
idx += 1
return lons_ring, lats_ring
def file(path,
pattern,
grep_args=None,
format_chained=True,
chained=None,
chained_status=None):
'''
Given a target ``path``, call ``grep`` to search for for ``pattern`` in that
file.
By default, the ``pattern`` and ``path`` will have ``.format()`` called on them with
``chained`` as the only argument. (So, use ``{0}`` in your pattern to
substitute the chained value.) If you want to avoid having to escape curly braces,
set ``format_chained=False``.
The first return value (status) will be True if the pattern is found, and
False othewise. The second argument will be the output of the ``grep``
command.
``grep_args`` can be used to pass in arguments to grep.
'''
if format_chained:
pattern = pattern.format(chained)
path = path.format(chained)
if grep_args is None:
grep_args = []
ret = _grep(pattern=pattern, path=path, *grep_args)
status = bool(ret)
return status, ret
def get_low_power_mode(self, port_num):
# Check for invalid port_num
if port_num < self.first_port or port_num > self.last_port:
return False
index = (port_num % 8)
i2c_index = (port_num / 8) + 1
path = self.lpmode_path
if i2c_index == 5:
return False
else:
port_path = path.format(self.port_to_i2cbus_mapping[i2c_index], (index + 1))
try:
reg_file = open(port_path)
except IOError as e:
print "Error: unable to open file: %s" % str(e)
return False
reg_value = reg_file.readline().rstrip()
reg_file.close()
if reg_value == '1':
return True
return False
def reset(self):
"""
Reset SFP and return all user module settings to their default srate.
Returns:
A boolean, True if successful, False if not
"""
cpld_index = (self.index / CPLD_PORT_NUM) + 1
index = (self.index % CPLD_PORT_NUM) + 1
if cpld_index == 5:
# Not support reset for SFP+
return True
else:
path = self.port_reset_path
port_path = path.format(self.port_to_i2cbus_mapping[cpld_index], index)
try:
reg_file = open(port_path, 'w')
except IOError as e:
print "Error: unable to open file: %s" % str(e)
return False
# reset
reg_file.write('1')
time.sleep(1)
reg_file.write('0')
reg_file.close()
return True
def closure():
optimizer.zero_grad()
vgg_model(input_image)
style_score = 0
content_score = 0
for s in style_losses:
style_score += s.loss
for c in content_losses:
content_score += c.loss
style_score *= style_weight
content_score *= content_weight
loss = style_score + content_score
loss.backward()
# puede darse que la imagen haya acabado con píxeles fuera de [0,1], esto lo arregla
input_image.data.clamp_(0, 1)
i[0] += 1
if i[0] % 50 == 0:
print("iteraciones: {}:".format(i[0]))
print(
'Diferencia de estilo : {:4f} Diferencia de contenido: {:4f}'
.format(style_score.item(), content_score.item()))
if path != "":
save(input_image, path.format(i[0]))
print()
return style_score + content_score
def run_style_transfer(content_image,
style_image,
input_image,
num_steps=300,
style_weight=1000000,
content_weight=1,
path=""):
vgg_model, style_losses, content_losses = get_style_model_and_losses(
style_image,
content_image,
style_layers=style_layers,
content_layers=content_layers)
optimizer = get_input_optimizer(input_image)
i = [0]
input_image.data.clamp_(0, 1)
if path != "":
path += "\\{}.jpg"
save(input_image, path.format(i[0]))
while i[0] <= num_steps:
def closure():
optimizer.zero_grad()
vgg_model(input_image)
style_score = 0
content_score = 0
for s in style_losses:
style_score += s.loss
for c in content_losses:
content_score += c.loss
style_score *= style_weight
content_score *= content_weight
loss = style_score + content_score
loss.backward()
# puede darse que la imagen haya acabado con píxeles fuera de [0,1], esto lo arregla
input_image.data.clamp_(0, 1)
i[0] += 1
if i[0] % 50 == 0:
print("iteraciones: {}:".format(i[0]))
print(
'Diferencia de estilo : {:4f} Diferencia de contenido: {:4f}'
.format(style_score.item(), content_score.item()))
if path != "":
save(input_image, path.format(i[0]))
print()
return style_score + content_score
optimizer.step(closure)
return input_image
def user_uploads_directory_path(instance, filename):
# file will be uploaded to MEDIA_ROOT/user_<id>/<datetime><fileext>
extension = os.path.splitext(filename)[1]
posted = instance.posted.strftime(r"%Y%m%d_%H%M%S")
path = 'uploads/user_{user_id}/{posted}{extension}'
return path.format(user_id=instance.user.id,
posted=posted,
extension=extension)
def _read_proc_file(path, pid, ignore_errors):
pid = pid or os.getpid()
try:
with open(path.format(pid=pid)) as f:
return f.read()
except IOError:
if not ignore_errors:
raise
def get_bounding_box(self):
"""Get the bounding box of this file."""
path = 'Data_Products/{file_group}/{file_group}_Gran_0/attr/'
prefix = path.format(**self.filetype_info)
lats = self.file_content[prefix + 'G-Ring_Latitude']
lons = self.file_content[prefix + 'G-Ring_Longitude']
return lons.ravel(), lats.ravel()
def input_paths_func(self):
if self.path is None:
return []
path_tmp = os.path.join(self.path, path.format("[0-9]"))
result = glob.glob(path_tmp)
for ext in other_extensions:
result += glob.glob(
os.path.join(self.path, set_extension(path_tmp, ext)))
return result
def __init__(self):
path = self.present_path
port_path = path.format(self.port_to_i2cbus_0[1], 1)
if not os.path.exists(port_path):
self.port_to_i2cbus_mapping = self.port_to_i2cbus_1
else:
self.port_to_i2cbus_mapping = self.port_to_i2cbus_0
for x in range(self.first_port, self.last_port + 1):
cpld_index = (x / 16) + 1
index = (x % 16) + 1
if cpld_index == 5:
path = self.eeprom_path_1
else:
path = self.eeprom_path
self.port_to_eeprom[x] = path.format(
self.port_to_i2cbus_mapping[cpld_index], index)
SfpUtilBase.__init__(self)
def github_path(i, chapter, j, section, k, notebook):
chapter = chapter.replace(" ", "_")
section = section.replace(" ", "_")
notebook = notebook.replace(" ", "_")
path = "https://github.com/gwoptics/learn_laser_interferometry/blob/master/{i:02d}_{chapter}/{j:02d}_{section}/{k:02d}_{notebook}.ipynb"
return path.format(i=i, j=j, k=k, chapter=chapter, section=section, notebook=notebook)
def resolve_path(self, path, root_config, anatomy=None):
if not root_config.get("root"):
root_config = {"root": root_config}
try:
return path.format(**root_config)
except KeyError:
msg = "Error in resolving remote root, unknown key"
log.error(msg)
def _get_core_freqs(core):
"""
Reads core's frequencies, sst-bf base, min scaling, max scaling
"""
freqs = {BF_PATH: None, MIN_PATH: None, MAX_PATH: None}
for path in freqs:
freqs[path] = _read_int_from_file(path.format(core))
return freqs[BF_PATH], freqs[MIN_PATH], freqs[MAX_PATH]
def get_log_file(k):
if config.has_section('logging'):
path = config.get('logging', 'path')
path = path.format(k)
if not os.path.exists(os.path.dirname(path)):
os.makedirs(os.path.dirname(path))
return open(path, 'a')
else:
return subprocess.PIPE
def get_log_file(k):
if config.has_section('logging'):
path = config.get('logging','path')
path = path.format(k)
if not os.path.exists(os.path.dirname(path)):
os.makedirs(os.path.dirname(path))
return open(path, 'a')
else:
return subprocess.PIPE
def absolute_uri(path="", *args):
from sentry.utils.http import absolute_uri
return absolute_uri(path.format(*args))
def testLoopOK1(self):
path_str = '/ISA_LOOP/GS_LOOP/ST_LOOP/DETAIL/2000A/2000B/2300/2400'
path = pyx12.path.X12Path(path_str)
self.assertEqual(path_str, path.format())
self.assertEqual(path.seg_id, None)
self.assertEqual(path.loop_list[2], 'ST_LOOP')
def testLoopSegOK1(self):
path_str = '/ISA_LOOP/GS_LOOP/ST_LOOP/DETAIL/2000A/2000B/2300/2400/SV2'
path = pyx12.path.X12Path(path_str)
self.assertEqual(path_str, path.format())
self.assertEqual(path.seg_id, 'SV2')
def test_Format3(self):
path_str = '/2000A/2000B/2300/2400/SV2[421]01'
path = pyx12.path.X12Path(path_str)
self.assertEqual(path_str, path.format())
def get_image_path(self, po):
path = settings.DIR_PORTAL_MAPS.rstrip("/") + "/{}_{}.jpg"
return path.format(po.guid, "image")
def start(self):
"""Start the node as a subprocess in a temporary directory.
"""
install_path = self.cluster.install_path
bin_path = os.path.join(self.working_path, "bin")
config_path = os.path.join(self.working_path, "config")
conf_path = os.path.join(config_path, "elasticsearch.yml")
log_path = os.path.join(self.working_path, "logs")
log_conf_path = os.path.join(config_path, "logging.yml")
data_path = os.path.join(self.working_path, "data")
# create temporary directory structure
for path in (bin_path, config_path, log_path, data_path):
if not os.path.exists(path):
os.mkdir(path)
# copy ES startup scripts
es_bin_dir = os.path.join(install_path, 'bin')
shutil.copy(os.path.join(es_bin_dir, 'elasticsearch'), bin_path)
shutil.copy(os.path.join(es_bin_dir, 'elasticsearch.in.sh'), bin_path)
# write configuration file
with open(conf_path, "w") as config:
config.write(CONF.format(
cluster_name=self.cluster.name,
node_name=self.name,
host=self.cluster.ip,
port=self.port,
tport=self.trans_port,
hosts=','.join(self.cluster.hosts),
working_path=self.working_path,
config_path=config_path,
data_path=data_path,
log_path=log_path,
))
# write log file
with open(log_conf_path, "w") as config:
config.write(LOG_CONF)
# create stdout/err files
self.stdout = tempfile.TemporaryFile(
suffix='stdout', dir=self.working_path)
self.stderr = tempfile.TemporaryFile(
suffix='stderr', dir=self.working_path)
# setup environment, copy from base process
environ = os.environ.copy()
# configure explicit ES_INCLUDE, to prevent fallback to
# system-wide locations like /usr/share, /usr/local/, ...
environ['ES_INCLUDE'] = os.path.join(bin_path, 'elasticsearch.in.sh')
lib_dir = os.path.join(install_path, 'lib')
# let the process find our jar files first
path = '{dir}/elasticsearch-*:{dir}/*:{dir}/sigar/*:$ES_CLASSPATH'
environ['ES_CLASSPATH'] = path.format(dir=lib_dir)
# reduce JVM startup time
environ['ES_MIN_MEM'] = '64m'
environ['JAVA_OPTS'] = \
'-client -XX:+TieredCompilation -XX:TieredStopAtLevel=1'
self.process = subprocess.Popen(
args=[bin_path + "/elasticsearch", "-f",
"-Des.config=" + conf_path],
stdout=self.stdout,
stderr=self.stderr,
env=environ
)
self.running = True
def get_map_path(self, po, typ):
path = settings.DIR_PORTAL_MAPS.rstrip("/") + "/{}_{}.png"
return path.format(po.guid, typ)
def format_path(path):
# format user directory in path
path = path.format(user=os.getenv('USERPROFILE') or os.getenv('HOME'))
return case_insensitive(path)
def write(self, yields, nuisances, path, fileFmt, signals = hwwsamples.signals ):
cardPath = path.format(mass = self._mass, bin = self._bin)
print 'Writing to '+cardPath
card = open( cardPath ,"w")
card.write('## Shape input card for H->WW analysis\n')
card.write('imax 1 number of channels\n')
card.write('jmax * number of background\n')
card.write('kmax * number of nuisance parameters\n')
card.write('-'*100+'\n')
card.write('bin %s' % self._bin+'\n')
if 'Data' not in yields:
self._log.warning( 'Yields: '+','.join(yields.keys()) )
raise RuntimeError('No Data found!')
card.write('observation %.0f\n' % yields['Data']._N)
# replace the second * with the bin?
if self._shape:
card.write('shapes * * '+
fileFmt.format(mass=self._mass, bin=self._bin)+
' histo_$PROCESS histo_$PROCESS_$SYSTEMATIC'+'\n')
card.write('shapes data_obs * '+
fileFmt.format(mass=self._mass, bin=self._bin)+
' histo_Data'+'\n')
card.write('-'*100+'\n')
bkgs = [ name for name in yields if name not in signals and name != 'Data']
sigs = [ name for name in yields if name in signals ]
keyline = []
keyline.extend([ (-i,s,yields[s]._N) for i,s in enumerate(sigs) ])
keyline.extend([ (i+1,b,yields[b]._N) for i,b in enumerate(bkgs) ])
coldef = 15
card.write('bin'.ljust(58)+''.join([self._bin.ljust(coldef)*len(keyline)])+'\n')
card.write('process'.ljust(58)+''.join([n.ljust(coldef) for (i,n,N) in keyline])+'\n' )
card.write('process'.ljust(58)+''.join([('%d' % i).ljust(coldef) for (i,n,N) in keyline])+'\n' )
card.write('rate'.ljust(58)+''.join([('%-.4f' % N).ljust(coldef) for (i,n,N) in keyline])+'\n' )
card.write('-'*100+'\n')
# nmax = max([len(n) for n in nuisances]
for name in nuisances:
(pdf,effect) = nuisances[name]
# if this nuisance has no effect on any sample, not even write it! --> "isAgoodNuisance"
isAgoodNuisance = False
for i,p,y in keyline:
if p in effect:
isAgoodNuisance = True
if 'WW' in signals and 'Gen_nlo_WW' in name:
isAgoodNuisance = False
if (isAgoodNuisance) :
if len(pdf) == 1: card.write('{0:<41} {1:<7} '.format(name,pdf[0]))
else: card.write('{0:<41} {1:<7} {2:<6} '.format(name,pdf[0],pdf[1]))
for i,p,y in keyline:
if p in effect:
#if 'FakeRate' in name:
if (pdf[0]=='gmN'): card.write('%-10.5f' % effect[p])
elif (pdf[0]=='shape' or pdf[0]=='shapeN2'): card.write('%-10d' % effect[p])
elif (isinstance(effect[p], tuple)): card.write('%.2f/%-5.2f' % (effect[p][0],effect[p][1]))
else: card.write('%-10.3f' % effect[p])
else: card.write('-'.ljust(coldef))
card.write('\n')
card.close()
