def prepare_alphafold_dataset(self, alpha):
print(f'Preparing dataset for alpha-{alpha}')
if alpha != self.ALPHAs:
self.gt_years = np.array2string(self.tl_model.years[(alpha-1)*self.PSI : alpha*self.PSI], separator='-')
else:
self.gt_years = np.array2string(self.tl_model.years[(alpha-1)*self.PSI : alpha*self.PSI+self.runoff_years], separator='-')
self.alpha_prepared_dir = str(Path(self.tl_model.prepared_data_dir) / f'alpha_{alpha}')
self.alpha_model_dir = str(Path(self.tl_model.cluster_dir) / f'alpha_{alpha}_GT-{self.gt_years}')
for pkl in utils.find(f'*alpha_{alpha}_preprocessed.pkl', self.alpha_prepared_dir):
if "target_ds_train" in pkl: self.target_ds_preprocessed_path = pkl
elif "rf_ds_train" in pkl: self.rf_ds_preprocessed_path = pkl
elif "target_ds_test" in pkl: self.x_test_path = pkl
elif "rf_ds_test" in pkl: self.y_test_path = pkl
LocalModelParams(self, utils.open_pickle(self.target_ds_preprocessed_path))
if utils.find('*standardized_stacked_arr.pkl', self.alpha_prepared_dir):
self.alpha_standardized_stacked_arr_path = utils.find(f'*standardized_stacked_arr.pkl', self.alpha_prepared_dir)[0]
else:
self.alpha_standardized_stacked_arr_path = prepare.flatten_and_standardize_dataset(self, self.alpha_prepared_dir)
print(f'--> Months for this dataset are: {self.month_names}')
print(
f'paths created @ prepare_alphafold_dataset():\nself.alpha_prepared_dir: "{self.alpha_prepared_dir}", \nself.alpha_model_dir: "{self.alpha_model_dir}"'
f'\nself.target_ds_preprocessed_path: "{self.target_ds_preprocessed_path}", \nself.rf_ds_preprocessed_path: "{self.rf_ds_preprocessed_path}"' \
f'\nself.rf_ds_preprocessed_path: "{self.rf_ds_preprocessed_path}", \nself.x_test_path: "{self.x_test_path}", \nself.y_test_path: "{self.y_test_path}"' \
f'\nself.alpha_standardized_stacked_arr_path: "{self.alpha_standardized_stacked_arr_path}", \nself.gt_years: {self.gt_years}' \
)
def brickfind_crawl(brick, args):
if brick.endswith("/"):
brick = brick[0:len(brick)-1]
working_dir = os.path.dirname(args.outfile)
mkdirp(working_dir, exit_on_err=True, logger=logger)
create_file(args.outfile, exit_on_err=True, logger=logger)
with open(args.outfile, "a+") as fout:
brick_path_len = len(brick)
def output_callback(path, filter_result):
path = path.strip()
path = path[brick_path_len+1:]
output_write(fout, path, args.output_prefix, encode=True)
ignore_dirs = [os.path.join(brick, dirname)
for dirname in
conf.get_opt("brick_ignore_dirs").split(",")]
find(brick, callback_func=output_callback,
ignore_dirs=ignore_dirs)
fout.flush()
os.fsync(fout.fileno())
def test_registration(browser):
# Data
page_heading_locator = 'form#register_form h2'
page_heading = 'Зарегистрироваться'
success_message = 'Спасибо за регистрацию!'
success_message_locator = '//div[contains(text(), "Спасибо за регистрацию")]'
# Arrange
browser.get(_locators.main_page_link)
utils.find(browser, _locators.login_link).click()
page_detector_text = utils.find(browser, page_heading_locator).text
assert page_heading in page_detector_text, \
"Search page heading '%s' should contain text '%s'" % (page_detector_text, page_heading)
# Steps
utils.registrate(browser, test_data.new_email, test_data.password)
WebDriverWait(browser, 5).until(
EC.presence_of_element_located((By.XPATH, success_message_locator)))
# Assert
success_message_text = utils.find_xpath(browser,
success_message_locator).text
assert success_message in success_message_text, \
"Search success message on page '%s'should contain text '%s'" % (success_message_text, success_message)
def detect_serialized_datasets(self):
"""
Finding raw data pickles.
If none found, proceed to creating pickles out of raw data.
calls -
1. prepare.get_raw_input_data
2. prepare.get_raw_target_data
3. prepare.prepare_dataset
"""
prepared_data_dir = str(utils.prepared_data_folder / self.dir_str / self.period)
os.makedirs(prepared_data_dir, exist_ok=True)
self.prepared_data_dir = prepared_data_dir
print(f'Looking for pickles in {self.prepared_data_dir}')
if len(utils.find('*serialized.pkl', self.prepared_data_dir)) == 2:
print('This domain-period combination has been serialized before, loading objects...')
for pkl in utils.find('*.pkl', self.prepared_data_dir):
if "input_ds" in pkl: self.input_ds_serialized_path = pkl
elif "rf_ds" in pkl: self.rf_ds_serialized_path = pkl
else:
print('Proceeding to load & serialize raw data. ')
self.raw_input_dir = prepare.get_raw_input_data(self)
self.raw_rf_dir = prepare.get_raw_target_data(self)
print(f'Raw input datasets taken from @: \n{self.raw_input_dir}')
print(f'Raw rainfall datasets taken from @: \n{self.raw_rf_dir}')
self.input_ds_serialized_path, self.rf_ds_serialized_path = prepare.prepare_dataset(self, self.prepared_data_dir)
print(f'Serialized raw input datasets @: \n{self.input_ds_serialized_path}')
print(f'Serialized raw RF datasets @: \n{self.rf_ds_serialized_path}')
def brickfind_crawl(brick, args):
if brick.endswith("/"):
brick = brick[0:len(brick)-1]
working_dir = os.path.dirname(args.outfile)
mkdirp(working_dir, exit_on_err=True, logger=logger)
create_file(args.outfile, exit_on_err=True, logger=logger)
with open(args.outfile, "a+") as fout:
brick_path_len = len(brick)
def output_callback(path, filter_result):
path = path.strip()
path = path[brick_path_len+1:]
output_write(fout, path, args.output_prefix,
encode=(not args.no_encode), tag=args.tag)
ignore_dirs = [os.path.join(brick, dirname)
for dirname in
conf.get_opt("brick_ignore_dirs").split(",")]
find(brick, callback_func=output_callback,
ignore_dirs=ignore_dirs)
fout.flush()
os.fsync(fout.fileno())
def brickfind_crawl(brick, args):
if brick.endswith("/"):
brick = brick[0:len(brick)-1]
working_dir = os.path.dirname(args.outfile)
mkdirp(working_dir, exit_on_err=True, logger=logger)
create_file(args.outfile, exit_on_err=True, logger=logger)
with open(args.outfile, "a+") as fout:
brick_path_len = len(brick)
def output_callback(path, filter_result, is_dir):
path = path.strip()
path = path[brick_path_len+1:]
if args.type == "both":
output_write(fout, path, args.output_prefix,
encode=(not args.no_encode), tag=args.tag,
field_separator=args.field_separator)
else:
if (is_dir and args.type == "d") or (
(not is_dir) and args.type == "f"):
output_write(fout, path, args.output_prefix,
encode=(not args.no_encode), tag=args.tag,
field_separator=args.field_separator)
ignore_dirs = [os.path.join(brick, dirname)
for dirname in
conf.get_opt("brick_ignore_dirs").split(",")]
find(brick, callback_func=output_callback,
ignore_dirs=ignore_dirs)
fout.flush()
os.fsync(fout.fileno())
def test_authorization(browser):
# Data
auth_heading = 'Войти'
page_auth_locator = 'form#login_form h2'
# Arrange
browser.get(_locators.main_page_link)
utils.find(browser, _locators.login_link).click()
page_auth_detector = utils.find(browser, page_auth_locator).text
assert auth_heading in page_auth_detector, \
"Search heading '%s' should contain text '%s'" % (page_auth_detector, auth_heading)
# Steps
utils.authorizate(browser, test_data.email, test_data.password)
WebDriverWait(browser, 5).until(
EC.presence_of_element_located(
(By.XPATH, _locators.success_message_locator)))
# Assert
success_message_text = utils.find_xpath(
browser, _locators.success_message_locator).text
assert _locators.success_message in success_message_text, \
"Search success message '%s' should contain text '%s'" % (success_message_text, _locators.success_message)
def gfid_to_path_using_batchfind(brick, gfids_file, output_file):
"""
find -samefile gets the inode number and crawls entire namespace
to get the list of files/dirs having same inode number.
Do find without any option, except the ignore directory option,
print the output in <INODE_NUM> <PATH> format, use this output
to look into in-memory dictionary of inode numbers got from the
list of GFIDs
"""
with open(output_file, "a+") as fout:
inode_dict = {}
with open(gfids_file) as f:
for gfid in f:
gfid = gfid.strip()
backend_path = os.path.join(brick, ".glusterfs",
gfid[0:2], gfid[2:4], gfid)
try:
inode_dict[str(os.stat(backend_path).st_ino)] = 1
except (IOError, OSError) as e:
if e.errno == ENOENT:
continue
else:
fail("%s Failed to convert to path from "
"GFID %s: %s" % (brick, gfid, e), logger=logger)
if not inode_dict:
return
def inode_filter(path):
try:
st = os.lstat(path)
except (OSError, IOError) as e:
if e.errno == ENOENT:
st = None
else:
raise
if st and inode_dict.get(str(st.st_ino), None):
return True
return False
brick_path_len = len(brick)
def output_callback(path):
path = path.strip()
path = path[brick_path_len+1:]
output_write(fout, path, args.output_prefix)
ignore_dirs = [os.path.join(brick, dirname)
for dirname in
conf.get_opt("brick_ignore_dirs").split(",")]
# Length of brick path, to remove from output path
find(brick, callback_func=output_callback,
filter_func=inode_filter,
ignore_dirs=ignore_dirs)
fout.flush()
os.fsync(fout.fileno())
def test_find():
looking_for = {'a': 1, 'match': True}
foo = [looking_for, {'b': 1}, {'a': 2}]
match = find(foo, {'a': 1})
assert match == looking_for
match = find(foo, {'c': 1})
assert not match
def filter(self, pts2D):
""" Filter image points based on loaded mask.
Usage: Mask = BaseMask('mask.png')
FilteredPts = Mask.filter(pts)
Input:
pts - 2-by-N numpy array of 2D points
Output:
FilteredPts - 2-by-N numpy array of 2D points. Those that agree
with the mask (i.e., those points [x, y] s.t. mask[x,y] = True)
remain the same, while invalid points are zeroed.
"""
# Find points that are not zero in image space
idxPts = utils.find( np.logical_and(pts2D[0, :] > 0, pts2D[1,:] > 0) )
# Get rounded and flattened (1D) pixel indexes for our 2D points
idxPtsFlat = utils.ravel_index(pts2D[::-1, idxPts].astype(int), \
self.data.shape)
# Find invalid 2D points by comparing them to the mask
idxInvalidPts = utils.find( self.data.flat[idxPtsFlat] == 0 )
# Set invalid points to zero
pts2D[:, idxPts[idxInvalidPts]] = 0
return pts2D
def test_add_product_to_cart(browser):
# Data
success_add_message_locator = 'div.alertinner'
success_message = 'был добавлен в вашу корзину'
item_in_cart_locator = 'div.alertinner strong'
item_in_cart_name = 'h3 a'
# Arrange
browser.get(_locators.main_page_link)
utils.find_xpath(browser, _locators.catalog_link).click()
page_catalog_detector = utils.find(browser,
_locators.catalog_heading_locator).text
assert _locators.catalog_heading in page_catalog_detector, \
"Search heading '%s' should contain text '%s'" % (page_catalog_detector, _locators.catalog_heading)
# Steps
utils.click_add_to_cart(browser)
success_add_message = utils.find(browser, success_add_message_locator).text
assert success_message in success_add_message, \
"Search heading '%s' should contain text '%s'" % (success_add_message, success_message)
what_is_in_cart = utils.find(browser, item_in_cart_locator).text
utils.view_cart(browser)
cart = utils.find(browser, item_in_cart_name).text
# Assert
assert what_is_in_cart == cart, "Wrong item in the cart"
def parse_info_html(account, info_html, track_id):
"""
组织raw对象
:return:
"""
raw = ResumeRaw()
raw.trackId = track_id
raw.source = 'FIVE_ONE'
raw.content = info_html
raw.email = account['userName']
tree = xmlh.document_fromstring(info_html)
img_url = common_utils.find(
'.*?(/Candidate/ReadAttach.aspx\?UserID=\\d+).*?', info_html)
try:
submit_times = common_utils.find(
'投递时间：<span class="blue_txt">(.*?)</span>', info_html)
if submit_times:
raw.resumeSubmitTime = submit_times
update_times = tree.xpath('//*[@id="lblResumeUpdateTime"]/text()')
if update_times:
raw.resumeUpdateTime = str(update_times[0])
except Exception as e:
logger.error(e.message)
if img_url:
url = "http://ehire.51job.com/" + img_url
oss_addr = inbox_utils.save_mobile_imgs_to_oss(url, 2, track_id)
if oss_addr:
raw.avatarUrl = oss_addr
return raw
def prepare_nfold_datasets(self): # i.e. split into different train/ground-truth(test) dataset
"""
Take previously pre-processed datasets & split into
input: x_train, x_test,
target/RF: y_train, y_test
"""
for alpha in range(1, self.ALPHAs+1):
if alpha != self.ALPHAs:
gt_years = np.array2string(self.tl_model.years[(alpha-1)*self.PSI : alpha*self.PSI], separator='-')
else:
gt_years = np.array2string(self.tl_model.years[(alpha-1)*self.PSI : alpha*self.PSI+self.runoff_years], separator='-')
new_cluster_dir = str(Path(self.tl_model.cluster_dir) / f'alpha_{alpha}_GT-{gt_years}')
os.makedirs(new_cluster_dir, exist_ok=True)
new_prepared_data_dir = str(Path(self.tl_model.prepared_data_dir) / f'alpha_{alpha}')
os.makedirs(new_prepared_data_dir, exist_ok=True)
if utils.find(f'*alpha_{alpha}_preprocessed.pkl', new_prepared_data_dir) and utils.find(f'*alpha_{alpha}_standardized_stacked_arr.pkl', new_prepared_data_dir):
pass
else:
if not utils.find(f'*target*alpha_{alpha}_preprocessed.pkl', new_prepared_data_dir):
print(f"=> No input datasets pre-processed for alpha of {alpha}")
prepare.cut_target_dataset(self, alpha, new_prepared_data_dir)
if not utils.find(f'*rf*alpha_{alpha}_preprocessed.pkl', new_prepared_data_dir):
print(f"=> No rainfall datasets pre-processed for alpha of {alpha}")
prepare.cut_rf_dataset(self, alpha, new_prepared_data_dir)
print(f'Preprocessed pickles for alpha split {alpha} can be found @:\n{new_prepared_data_dir}')
def h2(self):
goal1_hd = 0
goal2_hd = 0
for row in range(self.max_row + 1):
for col in range(self.max_col + 1):
if (self.state[row][col] != 0):
if self.state[row][col] != Node.goal1[row][col]:
goal1_hd += 1
if self.state[row][col] != Node.goal2[row][col]:
goal2_hd += 1
if (goal1_hd < goal2_hd):
total_cost = 0
for row in range(self.max_row + 1):
for col in range(self.max_col + 1):
if self.state[row][col] != Node.goal1[row][col]:
goal_r, goal_c = find(Node.goal1, self.state[row][col])
total_cost += getLowestCost(row, col, goal_r, goal_c)
return total_cost
else:
total_cost = 0
for row in range(self.max_row + 1):
for col in range(self.max_col + 1):
if self.state[row][col] != Node.goal2[row][col]:
goal_r, goal_c = find(Node.goal2, self.state[row][col])
total_cost += getLowestCost(row, col, goal_r, goal_c)
return total_cost
def compile_scores(self):
if utils.find(f'*Brier_scores_weighted-avg_boxplot_with_bootstrapped_whole-model_mean*.png', self.alpha_general_dir): pass
else:
print('Plotting mean brier scores across all alphas...')
evaluation.mean_brier_scores_all_alphas(self)
if utils.find(f'*ROC_whole-model-mean_all-alphas_micro-avged*.png', self.alpha_general_dir): pass
else:
print('Plotting ROC across all alphas...')
evaluation.ROC_all_alphas(self)
if utils.find(f'*gridded_AUC_whole-model_v2*.png', self.alpha_general_dir): pass
else:
print('Plotting gridded AUC across all alphas...')
evaluation.gridded_AUC_all_alphas(self)
if utils.find(f'*gridded_brier_whole-model_v2*.png', self.alpha_general_dir): pass
else:
print('Plotting gridded brier scores across all alphas...')
evaluation.gridded_brier_all_alphas(self)
# print(f'DEBUGGING: {utils.time_now()} - Plotting gridded AUC across all alphas...')
# evaluation.gridded_AUC_all_alphas(self)
# print(f'DEBUGGING: {utils.time_now()} - Plotting gridded brier scores across all alphas...')
# evaluation.gridded_brier_all_alphas(self)
with open(f'{self.alpha_general_dir}/flag', 'w+') as flag: pass # write flag to signal evaluation completed
def detect_prepared_datasets(self):
"""
Pre-processing, including time-slicing, removal of NAs, stacking & standardizing.
calls -
1. prepare.preprocess_time_series
2. prepare.flatten_and_standardize_dataset`
"""
if utils.find('*target_ds_preprocessed.pkl', self.prepared_data_dir) and \
utils.find('*rf_ds_preprocessed.pkl', self.prepared_data_dir) and \
utils.find('*standardized_stacked_arr.pkl', self.prepared_data_dir):
print('Pickles (preprocessed) found.')
for pkl in utils.find('*preprocessed.pkl', self.prepared_data_dir):
if "target_ds" in pkl: self.target_ds_preprocessed_path = pkl
elif "rf_ds" in pkl: self.rf_ds_preprocessed_path = pkl
LocalModelParams(self, utils.open_pickle(self.target_ds_preprocessed_path))
for pkl in utils.find('*standardized_stacked_arr.pkl', self.prepared_data_dir):
self.standardized_stacked_arr_path = pkl
else:
print('Pickles of pre-processed data incomplete. Proceeding to load & process raw dataset pickles.')
self.target_ds_preprocessed_path, self.rf_ds_preprocessed_path = prepare.preprocess_time_series(self, self.prepared_data_dir, self.ALPHAs)
LocalModelParams(self, utils.open_pickle(self.target_ds_preprocessed_path)) # generate new local model params
self.standardized_stacked_arr_path = prepare.flatten_and_standardize_dataset(self, self.prepared_data_dir)
print(f'--> Months for this dataset are: {self.month_names}')
def start_one_job(account):
"""开启一个帐号的任务
:account dict : 帐号,含有cookie
"""
global proxy
track_id = str(uuid.uuid1())
url = "http://ehire.51job.com/Inbox/InboxRecentEngine.aspx?Style=1"
refer_url = "http://ehire.51job.com/Navigate.aspx?ShowTips=11&PwdComplexity=N"
proxy = common_utils.get_proxy()
list_html = conn_html(account,
url,
5,
refer_url=refer_url,
track_id=track_id)
# list_html = open('text_htl').read() # 测试
while True:
if list_html:
if 'login' == list_html:
# 需要登录
logger.error("出现登录页面 %s" % account['userName'])
return 'login'
else:
hidEngineCvlogIds = common_utils.find(
'<input name="hidEngineCvlogIds" type="hidden" id="hidEngineCvlogIds" value="(.*?)" />',
list_html)
__VIEWSTATE = common_utils.find(
'<input type="hidden" name="__VIEWSTATE" id="__VIEWSTATE" value="(.*?)" />',
list_html)
resume_ids = parse_list_html(list_html, track_id=track_id)
if 'none' == resume_ids:
logger.info("邮箱没有邮件了--%s" % account['userName'])
return 'over'
elif 'refer-login' == resume_ids:
logger.error("出现登录页面 %s" % account['userName'])
return 'login'
if resume_ids:
ids_for = list(resume_ids)
logger.info('简历个数: %s' % len(resume_ids))
for id in ids_for:
info_url = 'http://ehire.51job.com/Candidate/ResumeViewFolder.aspx?hidSeqID=%s&hidFolder=EMP' % id
flag = info_main(account, info_url, track_id)
# flag = True
if 'login' == flag:
logger.error("出现登录页面 %s" % account['userName'])
return 'login'
if not flag: # 失败?
resume_ids.remove(id)
# 测试
# resume_ids = ['9229836941', ]
data = get_refer_data(resume_ids, __VIEWSTATE,
hidEngineCvlogIds)
# list_html = refer_list_html(account, data, 4)
else: # 解析失败
logger.error("页面 未能解析出简历%s" % account['userName'])
return 'error'
else: # 解析失败
logger.error("出现错误页面 %s" % account['userName'])
return 'error'
def parse_logical_bgp_sessions(data):
peerings = []
for ls in find('collection.junos:logical-systems', data, default=[]):
for group in find('protocols.bgp.group', ls, default=[]):
parse_bgp_group(group, peerings)
return peerings
def gfid_to_path_using_batchfind(brick, changelog_data):
# If all the GFIDs converted using gfid_to_path_using_pgfid
if not changelog_data.inodegfid_exists({"converted": 0}):
return
def inode_filter(path):
# Looks in inodegfid table, if exists returns
# inode number else None
try:
st = os.lstat(path)
except (OSError, IOError):
st = None
if st and changelog_data.inodegfid_exists({"inode": st.st_ino}):
return st.st_ino
return None
# Length of brick path, to remove from output path
brick_path_len = len(brick)
def output_callback(path, inode):
# For each path found, encodes it and updates path1
# Also updates converted flag in inodegfid table as 1
path = path.strip()
path = path[brick_path_len + 1 :]
path = output_path_prepare(path, args.output_prefix)
changelog_data.append_path1(path, inode)
ignore_dirs = [os.path.join(brick, dirname) for dirname in conf.get_opt("brick_ignore_dirs").split(",")]
# Full Namespace Crawl
find(brick, callback_func=output_callback, filter_func=inode_filter, ignore_dirs=ignore_dirs)
def prepare_dataset(model, dest):
"""
- xr.open_mfdataset() = loading
- restricting to certain variables + "levels" of variables
- combining variables xarrays into one
- restricting to only between 1999 to 2019
- slicing domain dimensions up to required specs (i.e. model.LON_S, model.LON_N, etc...)
- slicing up to chosen period only
- pickling the datasets (both input & rainfall) & returning them
"""
# searching for raw data pickles
preloaded_input_pickles = utils.find('*.pkl', model.raw_input_dir)
if preloaded_input_pickles:
print('Preloaded raw INPUT data pickles found...')
ds_CHOSEN_VARS_renamed = utils.open_pickle(utils.find('*.pkl', model.raw_input_dir)[0])
else:
print('Creating pickles of raw input data...')
ds_CHOSEN_VARS_renamed = save_preloaded_raw_input_data(model)
preloaded_input_pickles = utils.find('*.pkl', model.raw_rf_dir)
if preloaded_input_pickles:
print('Preloaded raw rainfall data pickles found...')
ds_RAINFALL = utils.open_pickle(utils.find('*.pkl', model.raw_rf_dir)[0])
else:
print('Creating pickles of raw rainfall data...')
ds_RAINFALL = save_preloaded_raw_rf_data(model)
print("Proceeding to do preliminary data cleaning...")
ds_sliced = ds_CHOSEN_VARS_renamed.sel(
level=slice(np.min(model.unique_pressure_lvls),np.max(model.unique_pressure_lvls)),
lat=slice(model.LAT_N,model.LAT_S), lon=slice(model.LON_W,model.LON_E),
time=slice('1999', '2019'))
ds_sliced_rhum = ds_sliced.rhum
ds_sliced_rhum_no925 = ds_sliced_rhum.drop_sel({"level":925})
ds_sliced_uwnd_only = ds_sliced.uwnd
ds_sliced_vwnd_only = ds_sliced.vwnd
ds_combined_sliced = xr.merge([ds_sliced_rhum_no925, ds_sliced_uwnd_only, ds_sliced_vwnd_only], compat='override')
rf_ds_sliced = ds_RAINFALL.sel(lat=slice(model.LAT_S, model.LAT_N), lon=slice(model.LON_W,model.LON_E))
print('Pickling domain- & feature-constrained input & RF datasets...')
if model.period == "NE_mon":
input_ds = ds_combined_sliced.sel(time=is_NE_mon(ds_combined_sliced['time.month']))
rf_ds = rf_ds_sliced.sel(time=is_NE_mon(rf_ds_sliced['time.month']))
input_ds_serialized_path = utils.to_pickle('raw_input_ds_NE_mon_serialized', input_ds, dest)
rf_ds_serialized_path = utils.to_pickle('raw_rf_ds_NE_mon_serialized', rf_ds, dest)
return input_ds_serialized_path, rf_ds_serialized_path
elif model.period == "SW_mon":
input_ds = ds_combined_sliced.sel(time=is_SW_mon(ds_combined_sliced['time.month']))
rf_ds = rf_ds_sliced.sel(time=is_SW_mon(rf_ds_sliced['time.month']))
input_ds_serialized_path = utils.to_pickle('raw_input_ds_SW_mon_serialized', input_ds, dest)
rf_ds_serialized_path = utils.to_pickle('raw_rf_ds_SW_mon_serialized', rf_ds, dest)
return input_ds_serialized_path, rf_ds_serialized_path
elif model.period == "inter_mon":
input_ds = ds_combined_sliced.sel(time=is_inter_mon(ds_combined_sliced['time.month']))
rf_ds = rf_ds_sliced.sel(time=is_inter_mon(rf_ds_sliced['time.month']))
input_ds_serialized_path = utils.to_pickle('raw_input_ds_inter_mon_serialized', input_ds, dest)
rf_ds_serialized_path = utils.to_pickle('raw_rf_ds_inter_mon_serialized', rf_ds, dest)
return input_ds_serialized_path, rf_ds_serialized_path
def gfid_to_path_using_pgfid(brick, changelog_data, args):
"""
For all the pgfids collected, Converts to Path and
does readdir on those directories and looks up inodegfid
table for matching inode number.
"""
populate_pgfid_and_inodegfid(brick, changelog_data)
# If no GFIDs needs conversion to Path
if not changelog_data.inodegfid_exists({"converted": 0}):
return
def inode_filter(path):
# Looks in inodegfid table, if exists returns
# inode number else None
try:
st = os.lstat(path)
except (OSError, IOError):
st = None
if st and changelog_data.inodegfid_exists({"inode": st.st_ino}):
return st.st_ino
return None
# Length of brick path, to remove from output path
brick_path_len = len(brick)
def output_callback(path, inode):
# For each path found, encodes it and updates path1
# Also updates converted flag in inodegfid table as 1
path = path.strip()
path = path[brick_path_len + 1:]
path = output_path_prepare(path, args)
changelog_data.append_path1(path, inode)
changelog_data.inodegfid_update({"converted": 1}, {"inode": inode})
ignore_dirs = [
os.path.join(brick, dirname)
for dirname in conf.get_opt("brick_ignore_dirs").split(",")
]
for row in changelog_data.pgfid_get():
try:
path = symlink_gfid_to_path(brick, row[0])
find(os.path.join(brick, path),
callback_func=output_callback,
filter_func=inode_filter,
ignore_dirs=ignore_dirs,
subdirs_crawl=False)
except (IOError, OSError) as e:
logger.warn("Error converting to path: %s" % e)
continue
def train_SOM(self, alpha=None):
d_hp_dir_path = str(utils.models_dir / self.dir_hp_str)
self.d_hp_dir_path = d_hp_dir_path
os.makedirs(d_hp_dir_path, exist_ok=True)
if not utils.find(f'*extent_{self.dir_str}.png', self.d_hp_dir_path):
visualization.get_domain_geometry(self, self.d_hp_dir_path)
models_dir_path = str(utils.models_dir / self.dir_hp_str / self.period) + f'_{self.month_names_joined}'
os.makedirs(models_dir_path, exist_ok=True)
self.models_dir_path = models_dir_path
# utils.update_cfgfile('Paths', 'models_dir_path', self.models_dir_path)
if alpha:
destination = self.alpha_model_dir
arr_path = self.alpha_standardized_stacked_arr_path
prefix = f'alpha_{alpha}_'
prompt = f'< alpha-{alpha} >'
else:
destination = self.models_dir_path
arr_path = self.standardized_stacked_arr_path
prefix = ''
prompt = ''
print(f'Destination: "{destination}", arr_path: "{arr_path}", prefix: "{prefix}"')
if utils.find(f'*{prefix}som_model.pkl', destination):
print(f'{utils.time_now()} - SOM model trained before, skipping...')
self.som_model_path = utils.find(f'*{prefix}som_model.pkl', destination)[0]
else:
print(f'{utils.time_now()} - {prompt} No SOM model trained for {self.domain}, {self.period}, for {self.hyperparameters}, doing so now...')
standardized_stacked_arr = utils.open_pickle(arr_path)
sominitstarttime = timer(); print(f'{utils.time_now()} - Initializing MiniSom... ')
som = MiniSom(self.gridsize, self.gridsize, # square
standardized_stacked_arr.shape[1],
sigma=self.sigma, learning_rate=self.learning_rate,
neighborhood_function='gaussian', random_seed=self.random_seed)
"""
Note: initializing PCA for weights is faster (~1/2 hour), but for serialized arrays > 300mb,
chances are this will kill the RAM and halt the entire process.
"""
## try:
## som.pca_weights_init(standardized_stacked_arr)
## except MemoryError as e:
## print(f'Memory error has occured: \n{e}')
print(f"Initialization took {utils.time_since(sominitstarttime)}.\n")
trainingstarttime = timer(); print(f"{utils.time_now()} - Beginning training.")
getattr(som, self.training_mode)(standardized_stacked_arr, self.iterations, verbose=True)
q_error = np.round(som.quantization_error(standardized_stacked_arr), 2)
print(f"Training complete. Q error is {q_error}, time taken for training is {utils.time_since(trainingstarttime)}s\n")
if alpha: self.som_model_path = utils.to_pickle(f'{self.RUN_datetime}_{prefix}som_model', som, destination)
else: self.som_model_path = utils.to_pickle(f'{self.RUN_datetime}_{prefix}som_model', som, destination)
def gfid_to_path_using_pgfid(brick, changelog_data, args):
"""
For all the pgfids collected, Converts to Path and
does readdir on those directories and looks up inodegfid
table for matching inode number.
"""
populate_pgfid_and_inodegfid(brick, changelog_data)
# If no GFIDs needs conversion to Path
if not changelog_data.inodegfid_exists({"converted": 0}):
return
def inode_filter(path):
# Looks in inodegfid table, if exists returns
# inode number else None
try:
st = os.lstat(path)
except (OSError, IOError):
st = None
if st and changelog_data.inodegfid_exists({"inode": st.st_ino}):
return st.st_ino
return None
# Length of brick path, to remove from output path
brick_path_len = len(brick)
def output_callback(path, inode):
# For each path found, encodes it and updates path1
# Also updates converted flag in inodegfid table as 1
path = path.strip()
path = path[brick_path_len+1:]
path = output_path_prepare(path, args)
changelog_data.append_path1(path, inode)
changelog_data.inodegfid_update({"converted": 1}, {"inode": inode})
ignore_dirs = [os.path.join(brick, dirname)
for dirname in
conf.get_opt("brick_ignore_dirs").split(",")]
for row in changelog_data.pgfid_get():
try:
path = symlink_gfid_to_path(brick, row[0])
find(os.path.join(brick, path),
callback_func=output_callback,
filter_func=inode_filter,
ignore_dirs=ignore_dirs,
subdirs_crawl=False)
except (IOError, OSError) as e:
logger.warn("Error converting to path: %s" % e)
continue
def parse_logical_interfaces(data, interfaces):
# make interface map for quick lookups
if_map = { i.name: i for i in interfaces }
for ls in find('collection.junos:logical-systems', data, default=[]):
for i in find('interfaces.interface', ls, default=[]):
iface = parse_interface(i, logical_system=ls['name'])
if iface.name in if_map:
if_map[iface.name].unitdict += iface.unitdict
else:
interfaces.append(iface)
return interfaces
def test_simple_nested(self):
data = {
'test': {
'hest': {
'test': 'best'
}
}
}
self.assertEqual(find('test.hest.test', data), 'best')
self.assertIn('test', find('test.hest', data))
def main():
aunts = parse(sys.stdin.readlines())
similar = lambda ref, x: all(x.get(k, v) == v for k, v in ref.items())
best_aunt = find(partial(similar, REF_AUNT), aunts)
print('part 1:', best_aunt['id'])
key2cmp = {'cats': ge, 'trees': ge, 'pomeranians': le, 'goldfish': le}
compare = lambda key, fst, snd: key2cmp.get(key, eq)(fst, snd)
similar = lambda ref, x: all(compare(k, x.get(k, v), v) for k, v in ref.items())
best_aunt = find(partial(similar, REF_AUNT), aunts)
print('part 2:', best_aunt['id'])
def parse_router(data, chassis_data=None):
router_data = data['tailf-ncs:device']
router = Router()
name = hostname_clean(router_data['address'])
router.name = name
router.version = find('config.junos:configuration.version', router_data)
if chassis_data:
chassis = find('junos-rpc:output.chassis-inventory.chassis', chassis_data, default={})
if chassis:
router.model = chassis['description']
router.hardware = parse_chassis(chassis)
return router
def main():
aunts = parse(sys.stdin.readlines())
similar = lambda ref, x: all(x.get(k, v) == v for k, v in ref.items())
best_aunt = find(partial(similar, REF_AUNT), aunts)
print('part 1:', best_aunt['id'])
key2cmp = {'cats': ge, 'trees': ge, 'pomeranians': le, 'goldfish': le}
compare = lambda key, fst, snd: key2cmp.get(key, eq)(fst, snd)
similar = lambda ref, x: all(
compare(k, x.get(k, v), v) for k, v in ref.items())
best_aunt = find(partial(similar, REF_AUNT), aunts)
print('part 2:', best_aunt['id'])
def parse_interface(item, logical_system=None):
iface = Interface()
iface.name = item['name']
iface.description = item.get('description')
iface.vlantagging = 'vlan-tagging' in item or 'flexible-vlan-tagging' in item
iface.unitdict = [parse_unit(u, logical_system) for u in item.get('unit', [])]
iface.bundle = find_first('bundle', item) or None
iface.tunneldict = [
{
'source': find('tunnel.source', u),
'destination': find('tunnel.destination', u)
} for u in item.get('unit', []) if 'tunnel' in u
]
return iface
def h1(self):
total_cost1 = 0
total_cost2 = 0
for row in range(self.max_row + 1):
for col in range(self.max_col + 1):
if self.state[row][col] != Node.goal1[row][col]:
goal_r, goal_c = find(Node.goal1, self.state[row][col])
total_cost1 += getLowestCost(row, col, goal_r, goal_c)
if self.state[row][col] != Node.goal2[row][col]:
goal_r, goal_c = find(Node.goal2, self.state[row][col])
total_cost2 += getLowestCost(row, col, goal_r, goal_c)
return min(total_cost1, total_cost2)
def findType(location,
bounds,
data,
allData,
placeType,
writeYmls=False,
useCached=False):
print('findType ' + str(placeType) + ' cached=' + str(useCached))
typeData = {}
allTypeData = []
if (useCached):
try:
stream = open(criterionName + '.' + placeType + '.all.yml', 'r')
cachedData = load(stream, Loader=Loader)
stream.close()
except Exception as e:
print(str(e))
raise Exception('ERROR: Failed to load yaml file ' + placeType +
'.all.yml')
allTypeData = cachedData
for p in cachedData:
gRec = {
'name': p['name'],
'vicinity': p['vicinity'],
'location': p['geometry']['location']
}
typeData[p['place_id']] = gRec
else:
utils.find(location,
bounds,
typeData,
allTypeData,
placeType=placeType,
doSplit=True)
data.update(typeData)
allData += allTypeData
if (writeYmls):
with open(criterionName + '.' + placeType + '.yml', 'w') as yaml_file:
dump(typeData, yaml_file, default_flow_style=False, Dumper=Dumper)
if (not useCached):
with open(criterionName + '.' + placeType + '.all.yml',
'w') as yaml_file:
dump(allTypeData,
yaml_file,
default_flow_style=False,
Dumper=Dumper)
def test_button_add_to_cart(browser):
browser.get(_locators.main_page_link)
utils.find_xpath(browser, _locators.catalog_link).click()
WebDriverWait(browser, 10).until(
EC.presence_of_element_located(
(By.CSS_SELECTOR, _locators.book_locator)))
utils.find(browser, _locators.book_locator).click()
WebDriverWait(browser, 10).until(
EC.presence_of_element_located(
(By.XPATH, _locators.button_add_to_cart)))
button_to_add = browser.find_element_by_xpath(_locators.button_add_to_cart)
assert button_to_add is not None, "There is no button(("
def get_arch_srcs(prebuilt, variant):
"""Returns build rule for arch specific srcs.
e.g.,
arch: {
arm: {
srcs: ["..."]
},
arm64: {
srcs: ["..."]
},
}
Args:
prebuilt: string, name of prebuilt object
variant: string, VNDK snapshot variant (e.g. 'arm64')
"""
arch_srcs = '{ind}arch: {{\n'.format(ind=self.INDENT)
variant_path = os.path.join(self._install_dir, variant)
src_paths = utils.find(variant_path, [prebuilt])
for src in sorted(src_paths):
arch_srcs += (
'{ind}{ind}{arch}: {{\n'
'{ind}{ind}{ind}srcs: ["{src}"],\n'
'{ind}{ind}}},\n'.format(
ind=self.INDENT,
arch=utils.arch_from_path(os.path.join(variant, src)),
src=src))
arch_srcs += '{ind}}},\n'.format(ind=self.INDENT)
return arch_srcs
def main():
with open(find('setup_agriculture.json', '/')) as f:
setup = json.load(f)
local_storage: str = setup.get('local_storage')
logging.basicConfig(
filename=os.path.join(local_storage, 'log.log'),
level=logging.WARNING,
format='%(asctime)s %(levelname)s %(name)s %(message)s')
temperature_port: int = setup['temperature'].get('temperature_port')
period: int = setup['temperature'].get('period')
wait: float = setup['temperature'].get('wait')
temperature_sensor = Temperature(temperature_port)
filename = os.path.join(local_storage,
'temperature_' + str(date.today()) + '.txt')
if not os.path.exists(filename):
with open(filename, 'w+') as f:
f.write('Timestamp, Temperature\n')
while exit_on_time(setup['temperature'].get('exit_time')):
measurement = catch_measurement(sensor=temperature_sensor,
period=period,
wait=wait)
save_measurement(measurement=measurement, path=filename)
quit()
def get_or_create_test_host(c):
instances = all_hosts(c)
test_host = find(instances, {'tags': {'name': 'test_host', 'type': 'skipper'}})
if not test_host:
test_host = create_host(c)
test_host.add_tag('name', 'test_host')
return test_host
def write_split_csv(save_dir, csv_dir, train_percent=0.8, test_percent=0.1):
os.makedirs(save_dir, exist_ok=True)
slides = find('*.csv', csv_dir)
random.shuffle(slides)
test_num = min(1, int(test_percent * len(slides)))
test_slides = slides[:test_num]
train_valid_slides = slides[test_num:]
test_set = pd.DataFrame()
for s in test_slides:
test_set = test_set.append(pd.read_csv(s))
train_valid_set = pd.DataFrame()
for s in train_valid_slides:
train_valid_set = train_valid_set.append(pd.read_csv(s))
train_valid_shuffled = train_valid_set.sample(frac=1)
# Calculate index for split 80:20 ratio
train_valid_index = round(len(train_valid_shuffled) * train_percent)
# Split into training and test sets
train_set = train_valid_shuffled[:train_valid_index].reset_index(drop=True)
valid_set = train_valid_shuffled[train_valid_index:].reset_index(drop=True)
train_set.to_csv(os.path.join(save_dir, 'train-compress.csv'), index=False)
test_set.to_csv(os.path.join(save_dir, 'test-compress.csv'), index=False)
valid_set.to_csv(os.path.join(save_dir, 'valid-compress.csv'), index=False)
def request_process_info(self,regex=None):
# info_list = [download.info_model() for download in self.processes]
if regex:
if find(regex , self.processes[-1].file_name):
self.downloadInfo_returned.emit(self.processes[-1].info_model())
else:
self.downloadInfo_returned.emit(self.processes[-1].info_model())
def get_arch_srcs(prebuilt, arch):
"""Returns build rule for arch specific srcs.
e.g.,
arch: {
arm: {
srcs: ["..."]
},
arm64: {
srcs: ["..."]
},
}
Args:
prebuilt: string, name of prebuilt object
arch: string, VNDK snapshot arch (e.g. 'arm64')
"""
arch_srcs = '{ind}arch: {{\n'.format(ind=self.INDENT)
src_paths = utils.find(src_root, [prebuilt])
# filter out paths under 'binder32' subdirectory
src_paths = filter(lambda src: not src.startswith(utils.BINDER32),
src_paths)
for src in sorted(src_paths):
arch_srcs += ('{ind}{ind}{arch}: {{\n'
'{ind}{ind}{ind}srcs: ["{src}"],\n'
'{ind}{ind}}},\n'.format(
ind=self.INDENT,
arch=utils.prebuilt_arch_from_path(
os.path.join(arch, src)),
src=src))
arch_srcs += '{ind}}},\n'.format(ind=self.INDENT)
return arch_srcs
def get_arch_srcs(prebuilt, arch):
"""Returns build rule for arch specific srcs.
e.g.,
arch: {
arm: {
srcs: ["..."]
},
arm64: {
srcs: ["..."]
},
}
Args:
prebuilt: string, name of prebuilt object
arch: string, VNDK snapshot arch (e.g. 'arm64')
"""
arch_srcs = '{ind}arch: {{\n'.format(ind=self.INDENT)
src_paths = utils.find(src_root, [prebuilt])
# filter out paths under 'binder32' subdirectory
src_paths = filter(lambda src: not src.startswith(utils.BINDER32),
src_paths)
for src in sorted(src_paths):
arch_srcs += ('{ind}{ind}{arch}: {{\n'
'{ind}{ind}{ind}srcs: ["{src}"],\n'
'{ind}{ind}}},\n'.format(
ind=self.INDENT,
arch=utils.prebuilt_arch_from_path(
os.path.join(arch, src)),
src=src))
arch_srcs += '{ind}}},\n'.format(ind=self.INDENT)
return arch_srcs
def rate_matrix(q,koffs,verbose=False):
"""Generate the stochastic rate matrix for the givens system."""
# Chromosome states can be represented by binary numerals; order the
# states this way.
G = len(koffs)
states = enumerate_states(G,q)
num_states = len(states)
assert len(states) == sum(choose(G,i) for i in range(q+1))
R = np.zeros((num_states,num_states))
for i,state_i in enumerate(states):
for j,state_j in enumerate(states):
if verbose:
print "considering:",i,state_i,"->",j,state_j
dist = hamming(state_i,state_j)
if dist != 1:
# deal with diagonal elements later...
if verbose:
print "distance is:",dist,"continuing..."
continue
if sum(state_j) == sum(state_i) + 1:
R[i][j] = q - sum(state_i)
if verbose:
print i,state_i,"->",j,state_j, "is an on-reaction, rate:",R[i][j]
elif sum(state_j) == sum(state_i) - 1:
diff_idx,diff_site = find(lambda (idx,(si,sj)):si != sj,enumerate(zip(state_i,state_j)))
R[i][j] = koffs[diff_idx]
if verbose:
print i,state_i,"->",j,state_j, "is an off-reaction (at site",diff_idx,") rate:",R[i][j]
# deal with diagonal elements
for i in range(num_states):
R[i][i] = -sum(R[i])
print "finished rate matrix"
return R
def dl_fna(species_name):
"""Dl fna if necessary, return filename"""
accession = dl_gbk(species_name)
print "accession:",accession
fna_name = accession + ".fna"
print "fna_name:",fna_name
target_path = os.path.join("data",species_name,fna_name)
if os.path.isfile(target_path):
print "found fna:",target_path
return target_path
print "didn't find fna for:",species_name,"downloading"
host.chdir('/genomes/Bacteria/')
dir_list = host.listdir(host.curdir)
sorted_dir_list = sorted(dir_list,key=lambda fname:levenshtein(species_name,fname))
for dir_name in sorted_dir_list:
print "trying:",dir_name
try:
host.chdir('/genomes/Bacteria/' + dir_name + '/')
sub_dir_list = host.listdir(host.curdir)
if find(lambda name:name.startswith(accession),sub_dir_list):
host.download(fna_name,target_path)
return target_path
except:
continue
print "Couldn't find fna for:",species_name
return None
def get(self, event_code, talk_code):
logging.info('Event Code: %s Talk Code: %s ' % (event_code, talk_code))
talk_code = talk_code.upper()
event_code = event_code.lower()
event = utils.find(lambda e: event_code == e.code, Events().get_all())
talk = Talks().get_talk_for_event(event_code, talk_code)
if((talk is None) or (event is None)):
self.response.write('Resource Not Found')
self.response.set_status(404)
return
talk = talk[0]
logging.info('Talk Details: ' + talk.code)
feedbacks = get_talk_feedbacks(talk_code)
template_values = {}
template_values['feedbacks'] = feedbacks
template_values['talk'] = talk
template_values['header'] = event.header
template_values['feedbacksCount'] = len(feedbacks)
template_values['averageRating'] = 0 if len(feedbacks) == 0 else sum(
[int(ast.literal_eval(feedback.content)['ratingC']) for feedback in feedbacks]) / len(feedbacks)
template = jinja_environment.get_template(html_path + 'view.html')
self.response.out.write(template.render(template_values))
def get_or_create_security_group(c):
groups = c.get_all_security_groups()
group = find(groups, {'name': 'skipper'})
if not group:
print("Creating new security group.")
group = c.create_security_group("skipper", "Skipper security group")
else:
print("Found existing security group.")
return group
def brickfind_crawl(brick, args):
if brick.endswith("/"):
brick = brick[0:len(brick)-1]
working_dir = os.path.dirname(args.outfile)
mkdirp(working_dir, exit_on_err=True, logger=logger)
create_file(args.outfile, exit_on_err=True, logger=logger)
with open(args.outfile, "a+") as fout:
brick_path_len = len(brick)
def mtime_filter(path):
try:
st = os.lstat(path)
except (OSError, IOError) as e:
if e.errno == ENOENT:
st = None
else:
raise
if st and (st.st_mtime > args.start or st.st_ctime > args.start):
return True
return False
def output_callback(path):
path = path.strip()
path = path[brick_path_len+1:]
output_write(fout, path, args.output_prefix)
ignore_dirs = [os.path.join(brick, dirname)
for dirname in
conf.get_opt("brick_ignore_dirs").split(",")]
if args.full:
find(brick, callback_func=output_callback,
ignore_dirs=ignore_dirs)
else:
find(brick, callback_func=output_callback,
filter_func=mtime_filter,
ignore_dirs=ignore_dirs)
fout.flush()
os.fsync(fout.fileno())
def search_files(request):
from utils import find
query = request.GET.get('query',None)
results=[]
if query:
shares = Share.user_queryset(request.user)
for s in shares:
r=find(s,query,prepend_share_id=False)
results.append({'share':s,'results':r})
return render(request, 'search/search_files.html', {'query':query,'results':results})
def getCommandInstance(command):
try:
module = importCommand(command)
commandInstances = [commandClass() for commandClass in Command.__subclasses__()]
commandToExec = utils.find(lambda recipes_installedCommand: recipes_installedCommand.name == command,
commandInstances)
except ImportError, e:
logging.error('Unknown command: %s.' % command)
print 'Search available commands with "ale search", install new command with "ale install <command>"'
return
def __init__(self, filename, rootname, glade_dir="."):
"""Initialize a new instance.
'filename' is the name of the .glade file containing the UI hierarchy.
'rootname' is the name of the topmost widget to be loaded.
'gladeDir' is the name of the directory, relative to the Python
path, in which to search for `filename'."""
if glade_dir:
filename = os.path.join(glade_dir, filename)
self._glade_path_name = utils.find(filename)
gtk.glade.XML.__init__(self, self._glade_path_name, rootname)
self.root = self.get_widget(rootname)
def patch():
os.chdir(utils.mojo_root_dir)
utils.filter_file("build/landmines.py", lambda line: not "gyp" in line)
utils.commit("filter gyp out of build/landmines.py")
utils.filter_file("gpu/BUILD.gn", lambda line: not "//gpu/ipc" in line)
utils.commit("filter gpu/ipc out of gpu/BUILD.gn")
for p in utils.find(["*.patch"], os.path.dirname(os.path.realpath(__file__))):
print "applying patch %s" % os.path.basename(p)
utils.system(["git", "apply", p])
utils.commit("applied patch %s" % os.path.basename(p))
def _parse_lib_list(self, txt_filename):
"""Returns a map of VNDK library lists per VNDK snapshot arch.
Args:
txt_filename: string, name of snapshot config file
Returns:
dict, e.g. {'arm64': ['libfoo.so', 'libbar.so', ...], ...}
"""
lib_map = dict()
for txt_path in utils.find(self._install_dir, [txt_filename]):
arch = utils.snapshot_arch_from_path(txt_path)
abs_path_of_txt = os.path.join(self._install_dir, txt_path)
with open(abs_path_of_txt, 'r') as f:
lib_map[arch] = f.read().strip().split('\n')
return lib_map
def search_ok(evt):
text = appenv.search_dialog.FindWindowByName("what").Value
if text == "":
wx.MessageBox(u"Nelze vyhledávat, nebylo specifikováno hledané.", u"Nelze pokračovat", wx.ICON_ERROR, appenv.search_dialog)
else:
if appenv.search_dialog.FindWindowByName("re").Value:
pos = utils.find_by_regex(text, appenv.policko.Value, appenv.search_dialog.FindWindowByName("case").Value)
else:
pos = utils.find(text, appenv.policko.Value, appenv.search_dialog.FindWindowByName("case").Value)
if pos >= 0:
appenv.policko.InsertionPoint = pos
appenv.search_dialog.Hide()
elif pos == -1:
wx.MessageBox(u"Hledaný text nebyl nalezen.", u"Nenalezeno", wx.ICON_ERROR, appenv.search_dialog)
elif pos == -2:
wx.MessageDialog(u"Vložený regulární výraz není syntakticky správný.", u"Neplatný regulární výraz", wx.ICON_ERROR, appenv.search_dialog)
def get(self, event_code):
event_code = event_code.lower()
event = utils.find(lambda e: event_code == e.code, Events().get_all())
if(event is None):
self.response.write('Resource Not Found')
self.set_status(404)
return
template_values = {}
feedbacks = get_all_feedbacks()
template_values['feedbacks'] = feedbacks
template_values['feedbacksCount'] = len(feedbacks)
template = jinja_environment.get_template(html_path + 'summary.html')
self.response.out.write(template.render(template_values))
def _parse_module_paths(self):
"""Parses the module_path.txt files into a dictionary,
Returns:
module_paths: dict, e.g. {libfoo.so: some/path/here}
"""
module_paths = dict()
for file in utils.find(self._install_dir, [self.MODULE_PATHS_TXT]):
file_path = os.path.join(self._install_dir, file)
with open(file_path, 'r') as f:
for line in f.read().strip().split('\n'):
paths = line.split(' ')
if len(paths) > 1:
if paths[0] not in module_paths:
module_paths[paths[0]] = paths[1]
return module_paths
def get_notice_file(prebuilt):
"""Returns build rule for notice file (attribute 'notice').
Args:
prebuilt: string, name of prebuilt object
"""
notice = ''
notice_file_name = '{}.txt'.format(prebuilt)
notices_dir = os.path.join(self._install_dir,
utils.NOTICE_FILES_DIR_PATH)
notice_files = utils.find(notices_dir, [notice_file_name])
if len(notice_files) > 0:
notice = '{ind}notice: "{notice_file_path}",\n'.format(
ind=self.INDENT,
notice_file_path=os.path.join(
'..', utils.NOTICE_FILES_DIR_PATH, notice_files[0]))
return notice
def get(self, event_code):
logging.info('Requested Url:' + self.request.url)
event_code = event_code.lower()
event = utils.find(lambda e: event_code == e.code, Events().get_all())
if event is None:
self.response.write('Resource Not Found')
self.set_status(404)
return
template_values = {'header': event.header, 'categories': Events().get_categories_for_event(event_code),
'action': event_code}
logging.info('Html Path: ' + html_path + 'home.html')
template = jinja_environment.get_template(html_path + 'home.html')
self.response.out.write(template.render(template_values))
def getPts3D(self, idx = None, filter=True):
"""Transform from depth image to 3D point cloud.
Usage: pts3d = d_img.getPts3D(idx)
Input:
idx - Either 1d or 2d list of indexes to keep. If 2d
indexes, we assume it is a list of [x y] pixel positions. Given that
python uses [row col], the indexed array is pts3D[idx[1,:],
idx[0,:]].
Output:
pts - np.ndarray - 3-by-N ndarray of 3D points
"""
# We need to distinguish between 1d indexes and 2d indexes
# 1d-indexes
if idx is None:
p3 = self.getXYZ().reshape(self.width*self.height, \
3).T.view(np.ndarray)
elif idx.size == 0:
p3 = np.zeros((3,0))
elif len(idx.shape) == 1:
pts3D = self.getXYZ().reshape(self.width*self.height, \
3).view(np.ndarray)
p3 = pts3D[:, idx]
elif len(idx.shape) == 2:
# Is it a list of indexes, or a True/False boolean array?
if idx.shape == self.shape:
idx = utils.C_ind2sub(utils.find(idx), self.shape)
idx = idx[::-1,:] # We need [x y], we have [r c]
idx = idx.astype(int) # Ensure it's integers
pts3D = self.getXYZ().view(np.ndarray)
p3 = pts3D[idx[1,:], idx[0,:], :].T
if filter:
# Delete points with negative Z
valid_pts = p3[2,:] > 0
p3[:, valid_pts]
return p3
def getValidPtsInView(self, view, output_type="bool"):
""" Compute indexes for the valid 2D points in view 'view'
Usage: idxValidPts = model.getValidPtsInView(view)
Input:
view - Index of the view for which we want the valid point indexes.
output_type{'bool'} - 'bool' (bool array of valid/invalid pts) or
'idx' (indexes of the valid points).
Output:
idxValidPts - valid point indexes for view 'view', i.e. those
s.t. self.pts2D[:, idxValidPts, view] != 0
"""
idxBoolPts = np.logical_and(self.pts2D[0, :, view] > 0, self.pts2D[1, :, view] > 0)
if output_type == "bool":
return idxBoolPts
elif output_type == "idx":
return utils.find(idxBoolPts)
def patch(dest_dir, relative_patches_dir=os.curdir):
"""Applies the *.patch files in |relative_patches_dir|.
Args:
relative_patches_dir: A directory path relative to the current directory.
Defaults to the directory of this file.
Raises:
subprocess.CalledProcessError if the patch couldn't be applied.
"""
patches_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), relative_patches_dir)
assert os.path.isdir(patches_dir)
os.chdir(dest_dir)
for p in utils.find(["*.patch"], patches_dir):
print "applying patch %s" % os.path.basename(p)
try:
utils.system(["git", "apply", p])
utils.commit("applied patch %s" % os.path.basename(p))
except subprocess.CalledProcessError:
print "ERROR: patch %s failed to apply" % os.path.basename(p)
raise
def post(self, event_code):
logging.info('Post Action:' + event_code)
event_code = event_code.lower()
event = utils.find(lambda e: event_code == e.code, Events().get_all())
talk_code = self.request.get('talk_code') or 'CONF'
talk = Talks().get_talk_for_event(event_code, talk_code)
if ((event is None) or (talk is None)):
self.response.write('Resource Not Found')
self.response.set_status(404)
return
talk = talk[0]
template_values = {}
comment = template_values['comment'] = cgi.escape(self.request.get('comment'))
ratingA = template_values['ratingA'] = cgi.escape(self.request.get('ratingA'))
ratingB = template_values['ratingB'] = cgi.escape(self.request.get('ratingB'))
ratingC = template_values['ratingC'] = cgi.escape(self.request.get('ratingC'))
provider = template_values['provider'] = cgi.escape(self.request.get('provider')) or 'Anonymous'
template_values['talk_code'] = talk_code
template_values['header'] = event.header
template_values['title'] = talk.title
template_values['presenters'] = talk.presenters
logging.info('Talk Code: %s Posted Content: %s ' % (talk_code, comment))
feedbackContent = {'comment': comment,
'ratingA': ratingA,
'ratingB': ratingB,
'ratingC': ratingC}
save_feedback(json.dumps(feedbackContent), event_code, talk_code, provider)
template = jinja_environment.get_template(html_path + 'message.html')
self.response.out.write(template.render(template_values))
