def cmd_setup_slave(args): ''' Setup proper slave environment and connect to master server ''' if not args.export: cmd = ["mkt", "setup", "--no-dirs"] if args.yes: cmd.append("-y") if args.force: cmd.append("-f") subprocess.call(cmd) utils.init_config_file() section = utils.load_config_file() subprocess.call(["sudo", "mkdir", "-p", section['src']]) subprocess.call(["sudo", "chown", "-R", utils.username() + ":" + utils.group(), section['src']]) subprocess.call(["sudo", "mkdir", "-p", section['logs']]) subprocess.call(["sudo", "chown", "-R", utils.username() + ":" + utils.group(), section['logs']]) with tempfile.NamedTemporaryFile("w") as f: f.write(open("/etc/fstab").read()) export = args.hostname[0] + ":" + \ section['src'][:-1] + "\t" + section['src'][:-1] + \ "\t" + "nfs" + "\t" + "ro,nolock 0 0" + "\n" + \ args.hostname[0] + ":" + \ section['logs'][:-1] + "\t" + section['logs'][:-1] + \ "\t" + "nfs" + "\t" + "rw,nolock 0 0" + "\n" f.write(export) f.flush() subprocess.call(["sudo", "cp", f.name, "/etc/fstab"]) subprocess.call(["sudo", "mount", "-a", "-t", "nfs", "-o", "remount"])
def group_by_best_model(result): # select best model by max val acc result = utils.group(result, 5) result = utils.reduce_by_max_val(result) result = utils.group(result, 4, keepdim=True) result = utils.reduce_by_max_val(result, dim=4, dim_name="model") return result
def test(): import sys verbose = '-v' in sys.argv def assertEqual(a, b): if a == b: if verbose: sys.stderr.write('.') sys.stderr.flush() else: assert a == b, "\nexpected: %s\ngot: %s" % (repr(b), repr(a)) from utils import storage, group t = Template tests = [ lambda: t('1')(), '1\n', lambda: t('$def with ()\n1')(), '1\n', lambda: t('$def with (a)\n$a')(1), '1\n', lambda: t('$def with (a=0)\n$a')(1), '1\n', lambda: t('$def with (a=0)\n$a')(a=1), '1\n', lambda: t('$if 1: 1')(), '1\n', lambda: t('$if 1:\n 1')(), '1\n', lambda: t('$if 0: 0\n$elif 1: 1')(), '1\n', lambda: t('$if 0: 0\n$elif None: 0\n$else: 1')(), '1\n', lambda: t('$if (0 < 1) and (1 < 2): 1')(), '1\n', lambda: t('$for x in [1, 2, 3]: $x')(), '1\n2\n3\n', lambda: t('$for x in []: 0\n$else: 1')(), '1\n', lambda: t('$def with (a)\n$while a and a.pop(): 1')([1, 2, 3]), '1\n1\n1\n', lambda: t('$while 0: 0\n$else: 1')(), '1\n', lambda: t('$ a = 1\n$a')(), '1\n', lambda: t('$# 0')(), '', lambda: t('$def with (d)\n$for k, v in d.iteritems(): $k')({1: 1}), '1\n', lambda: t('$def with (a)\n$(a)')(1), '1\n', lambda: t('$def with (a)\n$a')(1), '1\n', lambda: t('$def with (a)\n$a.b')(storage(b=1)), '1\n', lambda: t('$def with (a)\n$a[0]')([1]), '1\n', lambda: t('${0 or 1}')(), '1\n', lambda: t('$ a = [1]\n$a[0]')(), '1\n', lambda: t('$ a = {1: 1}\n$a.keys()[0]')(), '1\n', lambda: t('$ a = []\n$if not a: 1')(), '1\n', lambda: t('$ a = {}\n$if not a: 1')(), '1\n', lambda: t('$ a = -1\n$a')(), '-1\n', lambda: t('$ a = "1"\n$a')(), '1\n', lambda: t('$if 1 is 1: 1')(), '1\n', lambda: t('$if not 0: 1')(), '1\n', lambda: t('$if 1:\n $if 1: 1')(), '1\n', lambda: t('$ a = 1\n$a')(), '1\n', lambda: t('$ a = 1.\n$a')(), '1.0\n', lambda: t('$({1: 1}.keys()[0])')(), '1\n', lambda: t('$for x in [1, 2, 3]:\n\t$x') (), ' 1\n 2\n 3\n', lambda: t('$def with (a)\n$:a')(1), '1\n', ] for func, value in group(tests, 2): assertEqual(func(), value) j = Template("$var foo: bar")() assertEqual(str(j), '') assertEqual(j.foo, 'bar\n') if verbose: sys.stderr.write('\n')
def bin_points(x_pnts, y_pnts, min_bucket, split_bucket): # adaptive min_bucket and split_bucket adaptive_min_bucket = isinstance(min_bucket, str) and \ min_bucket == 'adaptive' adaptive_split_bucket = isinstance(split_bucket, str) and \ split_bucket == 'adaptive' if adaptive_min_bucket or adaptive_split_bucket: contrast_range = x_pnts[[0, -1]] # or if we want to trim edges: # drop_elements = int(np.ceil(len(x_pnts) * 0.08)) # contrast_range = x_pnts[drop_elements:-drop_elements][[0, -1]] contrast_range = np.diff(contrast_range)[0] if adaptive_min_bucket: min_bucket = contrast_range / 20. if adaptive_split_bucket: split_bucket = contrast_range / 10. # look for buckets x_buckets = group(np.diff(x_pnts) <= min_bucket) n_pnts_in_bucket = (np.diff(x_buckets, axis=-1) + 1).ravel() good_buckets = n_pnts_in_bucket >= (3 - 1) # -1 because of diff if x_buckets.shape[0] > 0 and np.any(good_buckets): x_buckets = x_buckets[good_buckets, :] # turn buckets to slices, get mean and sem x_buckets[:, 1] += 2 # +1 because of python slicing # another +1 because of diff slices = [slice(l[0], l[1]) for l in x_buckets] # test each slice for contrast range and split if needed add_slices = list() ii = 0 while ii < len(slices): slc = slices[ii] pnts = x_pnts[slc] l, h = pnts[[0, -1]] rng = h - l start = slc.start if rng > split_bucket: slices.pop(ii) n_full_splits = int(np.floor(rng / split_bucket)) last_l = l last_ind = 0 for splt in range(n_full_splits): this_high = last_l + split_bucket this_ind = np.where(pnts >= this_high)[0][0] add_slices.append(slice(start + last_ind, start + this_ind + 1)) last_l = this_high last_ind = this_ind + 1 # last one - to the end if start + last_ind < slc.stop: add_slices.append(slice(start + last_ind, slc.stop)) else: ii += 1 slices.extend(add_slices) return slices
def find_elements(diff_array): diff_array.flags.writeable = True diff_array[22:92,160:320] = zeros((70,160)) contours,hierarchy = cv2.findContours(diff_array, 1, 2) bird = None raw_pipes = [] for cnt in contours: area = cv2.contourArea(cnt) rect = cv2.boundingRect(cnt) if 1400 < area < 1700 and 90 < rect[0] < 170: bird = rect elif area > 1500: raw_pipes.append(rect) pipes = [] for pipe in group(sorted(raw_pipes, key=lambda x:x[0]), 2): x = min(pipe[0][0], pipe[1][0]) y_values = [pipe[0][1], pipe[1][1]] min_y_index, min_y_value = min(enumerate(y_values), key=operator.itemgetter(1)) y = min_y_value + [pipe[0][3], pipe[1][3]][min_y_index] size_x = min(pipe[0][2], pipe[1][2]) size_y = y_values[min_y_index-1] - y_values[min_y_index] - y + min_y_value p = [x, y, size_x, size_y] pipes.append(p) return bird, pipes
def _match(self, mapping, value): for pat, what in utils.group(mapping, 2): rx = utils.re_compile('^' + pat + '$') result = rx.match(value) if result: return what, [x and urllib.unquote(x) for x in result.groups()] return None, None
def _match(self, mapping, value): for pat, what in utils.group(mapping, 2): rx = utils.re_compile("^" + pat + "$") result = rx.match(value) if result: return what, [x and urllib.unquote(x) for x in result.groups()] return None, None
def bin_points(x_pnts, y_pnts, min_bucket, split_bucket): # adaptive min_bucket and split_bucket adaptive_min_bucket = isinstance(min_bucket, str) and \ min_bucket == 'adaptive' adaptive_split_bucket = isinstance(split_bucket, str) and \ split_bucket == 'adaptive' if adaptive_min_bucket or adaptive_split_bucket: contrast_range = x_pnts[[0, -1]] # or if we want to trim edges: # drop_elements = int(np.ceil(len(x_pnts) * 0.08)) # contrast_range = x_pnts[drop_elements:-drop_elements][[0, -1]] contrast_range = np.diff(contrast_range)[0] if adaptive_min_bucket: min_bucket = contrast_range / 20. if adaptive_split_bucket: split_bucket = contrast_range / 10. # look for buckets x_buckets = group(np.diff(x_pnts) <= min_bucket) n_pnts_in_bucket = (np.diff(x_buckets, axis=-1) + 1).ravel() good_buckets = n_pnts_in_bucket >= (3 - 1) # -1 because of diff if x_buckets.shape[0] > 0 and np.any(good_buckets): x_buckets = x_buckets[good_buckets, :] # turn buckets to slices, get mean and sem x_buckets[:, 1] += 2 # +1 because of python slicing # another +1 because of diff slices = [slice(l[0], l[1]) for l in x_buckets] # test each slice for contrast range and split if needed add_slices = list() ii = 0 while ii < len(slices): slc = slices[ii] pnts = x_pnts[slc] l, h = pnts[[0, -1]] rng = h - l start = slc.start if rng > split_bucket: slices.pop(ii) n_full_splits = int(np.floor(rng / split_bucket)) last_l = l last_ind = 0 for splt in range(n_full_splits): this_high = last_l + split_bucket this_ind = np.where(pnts >= this_high)[0][0] add_slices.append( slice(start + last_ind, start + this_ind + 1)) last_l = this_high last_ind = this_ind + 1 # last one - to the end if start + last_ind < slc.stop: add_slices.append(slice(start + last_ind, slc.stop)) else: ii += 1 slices.extend(add_slices) return slices
def view_raw(args): data = read_raw(args.view_count) for name, _, cls, bbox in data: print(name, cls, bbox) for d in group(data, 36): # display raw images _, images, _, _ = zip(*d) show_image(merge_images_with_border([cv2.resize(i, (150, 150)) for i in images]))
def _match(self, mapping, value): for pat, what in utils.group(mapping, 2): if isinstance(what, basestring): what, result = utils.re_subm("^" + pat + "$", what, web.ctx.path) else: result = utils.re_compile("^" + pat + "$").match(web.ctx.path) if result: # it's a match return what, [x and urllib.unquote(x) for x in result.groups()] return None, None
def get_vector_carno(vector): vector = vector[::-1] arr = [0, 1, 3, 2, 4, 5, 7, 6, 12, 13, 15, 14, 8, 9, 11, 10] groups = group(arr, 4) lst = [['', 'cd', 'cD', 'CD', 'Cd']] rows = ['ab', 'aB', 'AB', 'Ab'] for i, grp in enumerate(groups): lst += [[rows[i]] + [vector[x] for x in grp]] table_view(lst, free_space_right=1)
def _match(self, mapping, value): for pat, what in utils.group(mapping, 2): if isinstance(what, basestring): what, result = utils.re_subm('^' + pat + '$', what, value) else: result = utils.re_compile('^' + pat + '$').match(value) if result: # it's a match return what, [x for x in result.groups()] return None, None
def create_listsoflists(): global tag_list tag_list = connection.list_tags() name_list = list() for tag in tag_list: name_list.append(tag.name) tag_grouped = utils.group(name_list, 3) return tag_grouped
def handle(self): for dir, what in utils.group(self.mapping, 2): if web.ctx.path.startswith(dir + '/'): # change paths to make path relative to dir web.ctx.home += dir web.ctx.homepath += dir web.ctx.path = web.ctx.path[len(dir):] web.ctx.fullpath = web.ctx.fullpath[len(dir):] return self._delegate(what, self.fvars) raise NotFound
def get_installment_payments(path, num_rows=None): """Preprocess and extract features from installments_payments. Arguments: path: Path to the folder where files are saved (string). num_rows: Number of rows to read; None reads all rows (int, default: None). Returns: df: DataFrame with processed data. """ pay = pd.read_csv(os.path.join(path, 'installments_payments.csv'), nrows=num_rows) # Group payments and get Payment difference pay = utils.do_sum(pay, ['SK_ID_PREV', 'NUM_INSTALMENT_NUMBER'], 'AMT_PAYMENT', 'AMT_PAYMENT_GROUPED') pay['PAYMENT_DIFFERENCE'] = pay['AMT_INSTALMENT'] - pay['AMT_PAYMENT_GROUPED'] pay['PAYMENT_RATIO'] = pay['AMT_INSTALMENT'] / pay['AMT_PAYMENT_GROUPED'] pay['PAID_OVER_AMOUNT'] = pay['AMT_PAYMENT'] - pay['AMT_INSTALMENT'] pay['PAID_OVER'] = (pay['PAID_OVER_AMOUNT'] > 0).astype(int) # Payment Entry: Days past due and Days before due pay['DPD'] = pay['DAYS_ENTRY_PAYMENT'] - pay['DAYS_INSTALMENT'] pay['DPD'] = pay['DPD'].apply(lambda x: 0 if x <= 0 else x) pay['DBD'] = pay['DAYS_INSTALMENT'] - pay['DAYS_ENTRY_PAYMENT'] pay['DBD'] = pay['DBD'].apply(lambda x: 0 if x <= 0 else x) # Flag late payment pay['LATE_PAYMENT'] = pay['DBD'].apply(lambda x: 1 if x > 0 else 0) # Percentage of payments that were late pay['INSTALMENT_PAYMENT_RATIO'] = pay['AMT_PAYMENT'] / pay['AMT_INSTALMENT'] pay['LATE_PAYMENT_RATIO'] = pay.apply(lambda x: x['INSTALMENT_PAYMENT_RATIO'] if x['LATE_PAYMENT'] == 1 else 0, axis=1) # Flag late payments that have a significant amount pay['SIGNIFICANT_LATE_PAYMENT'] = pay['LATE_PAYMENT_RATIO'].apply(lambda x: 1 if x > 0.05 else 0) # Flag k threshold late payments pay['DPD_7'] = pay['DPD'].apply(lambda x: 1 if x > 7 else 0) pay['DPD_15'] = pay['DPD'].apply(lambda x: 1 if x > 15 else 0) # Aggregations by SK_ID_CURR pay_agg = utils.group(pay, 'INS_', config.INSTALLMENTS_AGG) # Installments in the last x months for months in [18, 36]: recent_prev_id = pay[pay['DAYS_INSTALMENT'] >= -30*months]['SK_ID_PREV'].unique() pay_recent = pay[pay['SK_ID_PREV'].isin(recent_prev_id)] prefix = 'INS_{}M_'.format(months) pay_agg = utils.group_and_merge(pay_recent, pay_agg, prefix, config.INSTALLMENTS_TIME_AGG) # Last x periods trend features group_features = ['SK_ID_CURR', 'SK_ID_PREV', 'DPD', 'LATE_PAYMENT', 'PAID_OVER_AMOUNT', 'PAID_OVER', 'DAYS_INSTALMENT'] group = pay[group_features].groupby('SK_ID_CURR') func = partial(_trend_in_last_k_instalment_features, periods=[12, 24, 60, 120]) g = utils.parallel_apply(group, func, index_name='SK_ID_CURR', chunk_size=10000).reset_index() pay_agg = pay_agg.merge(g, on='SK_ID_CURR', how='left') # Last loan features g = utils.parallel_apply(group, _installments_last_loan_features, index_name='SK_ID_CURR', chunk_size=10000).reset_index() pay_agg = pay_agg.merge(g, on='SK_ID_CURR', how='left') return pay_agg
async def set_lang(self, chat_id, command, args): lang = args.strip() if args else None if lang: await self.save_user_settings(chat_id, {'lang': LANG.get(lang)}) await bot.sendMessage(chat_id, 'language saved', reply_markup=ReplyKeyboardRemove()) else: markup = ReplyKeyboardMarkup( keyboard=group(['/lang' + x for x in LANG.keys()], 2), one_time_keyboard=True ) await bot.sendMessage(chat_id, 'set language', reply_markup=markup)
async def set_cc(self, chat_id, command, args): cc = args.strip() if args else None if cc: await self.save_user_settings(chat_id, {'cc': CC.get(cc)}) await bot.sendMessage(chat_id, 'region saved', reply_markup=ReplyKeyboardRemove()) else: markup = ReplyKeyboardMarkup( keyboard=group(['/cc' + x for x in CC.keys()], 3), one_time_keyboard=True ) await bot.sendMessage(chat_id, 'set region', reply_markup=markup)
def get_pos_cash(path, num_rows= None): """Preprocess and extract features from POS_CASH_balance file. Arguments: path: Path to the folder where files are saved (string). num_rows: Number of rows to read; None reads all rows (int, default: None). Returns: df: DataFrame with processed data. """ pos = pd.read_csv(os.path.join(path, 'POS_CASH_balance.csv'), nrows=num_rows) pos, categorical_cols = utils.one_hot_encoder(pos, nan_as_category=False) # Flag months with late payment pos['LATE_PAYMENT'] = pos['SK_DPD'].apply(lambda x: 1 if x > 0 else 0) # Aggregate by SK_ID_CURR categorical_agg = {key: ['mean'] for key in categorical_cols} pos_agg = utils.group(pos, 'POS_', {**config.POS_CASH_AGG, **categorical_agg}) # Sort and group by SK_ID_PREV sort_pos = pos.sort_values(by=['SK_ID_PREV', 'MONTHS_BALANCE']) gp = sort_pos.groupby('SK_ID_PREV') df = pd.DataFrame() df['SK_ID_CURR'] = gp['SK_ID_CURR'].first() df['MONTHS_BALANCE_MAX'] = gp['MONTHS_BALANCE'].max() # Percentage of previous loans completed and completed before initial term df['POS_LOAN_COMPLETED_MEAN'] = gp['NAME_CONTRACT_STATUS_Completed'].mean() df['POS_COMPLETED_BEFORE_MEAN'] = gp['CNT_INSTALMENT'].first() - gp['CNT_INSTALMENT'].last() df['POS_COMPLETED_BEFORE_MEAN'] = df.apply(lambda x: 1 if x['POS_COMPLETED_BEFORE_MEAN'] > 0 and x['POS_LOAN_COMPLETED_MEAN'] > 0 else 0, axis=1) # Number of remaining installments (future installments) and percentage from total df['POS_REMAINING_INSTALMENTS'] = gp['CNT_INSTALMENT_FUTURE'].last() df['POS_REMAINING_INSTALMENTS_RATIO'] = gp['CNT_INSTALMENT_FUTURE'].last()/gp['CNT_INSTALMENT'].last() # Group by SK_ID_CURR and merge df_gp = df.groupby('SK_ID_CURR').sum().reset_index() df_gp.drop(['MONTHS_BALANCE_MAX'], axis=1, inplace= True) pos_agg = pd.merge(pos_agg, df_gp, on= 'SK_ID_CURR', how= 'left') del df, gp, df_gp, sort_pos gc.collect() # Percentage of late payments for the 3 most recent applications pos = utils.do_sum(pos, ['SK_ID_PREV'], 'LATE_PAYMENT', 'LATE_PAYMENT_SUM') # Last month of each application last_month_df = pos.groupby('SK_ID_PREV')['MONTHS_BALANCE'].idxmax() # Most recent applications (3) sort_pos = pos.sort_values(by=['SK_ID_PREV', 'MONTHS_BALANCE']) gp = sort_pos.iloc[last_month_df].groupby('SK_ID_CURR').tail(3) gp_mean = gp.groupby('SK_ID_CURR').mean().reset_index() pos_agg = pd.merge(pos_agg, gp_mean[['SK_ID_CURR','LATE_PAYMENT_SUM']], on='SK_ID_CURR', how='left') # Drop some useless categorical features drop_features = [ 'POS_NAME_CONTRACT_STATUS_Canceled_MEAN', 'POS_NAME_CONTRACT_STATUS_Amortized debt_MEAN', 'POS_NAME_CONTRACT_STATUS_XNA_MEAN'] pos_agg.drop(drop_features, axis=1, inplace=True) return pos_agg
async def set_cc(self, chat_id, command, args): cc = args.strip() if args else None if cc: await self.save_user_settings(chat_id, {'cc': CC.get(cc)}) await bot.sendMessage(chat_id, 'region saved', reply_markup=ReplyKeyboardRemove()) else: markup = ReplyKeyboardMarkup(keyboard=group( ['/cc' + x for x in CC.keys()], 3), one_time_keyboard=True) await bot.sendMessage(chat_id, 'set region', reply_markup=markup)
async def set_lang(self, chat_id, command, args): lang = args.strip() if args else None if lang: await self.save_user_settings(chat_id, {'lang': LANG.get(lang)}) await bot.sendMessage(chat_id, 'language saved', reply_markup=ReplyKeyboardRemove()) else: markup = ReplyKeyboardMarkup(keyboard=group( ['/lang' + x for x in LANG.keys()], 2), one_time_keyboard=True) await bot.sendMessage(chat_id, 'set language', reply_markup=markup)
def __call__(self, hits): words = getattr(self._cl.query, 'sphinx', self._cl.query) docs = [] for match in hits.matches: docs.extend([utils._unicode(match['@hit'][f]) for f in self._on_fields]) all_excerpts = self._cl.BuildExcerpts(docs, self._index, words, self._opts) print all_excerpts for match, excerpts in zip(hits.matches, utils.group(all_excerpts, len(self._on_fields))): for f, excerpt in zip(self._on_fields, excerpts): match['@hit'][f + self._suffix] = excerpt or match['@hit'][f]
def get_shuffle_tokens(self): #shuffle_keys = [] #for s in string.lowercase: # for s2 in string.lowercase: # shuffle_keys.append(s+s2) #for s in string.uppercase: # for s2 in string.lowercase+string.uppercase: # shuffle_keys.append(s+s2) # #return dict([(('shuffle', shard), Token()) for shard in shuffle_keys]) # return dict([(('shuffle', shard), Token()) for shard in string.lowercase+string.uppercase]) return dict([(('shuffle', tuple(g)), Token()) for g in group(list(self.shuffle_keys), ShuffleKeysPerShard)])
def handle(mapping, fvars=None): """ Call the appropriate function based on the url to function mapping in `mapping`. If no module for the function is specified, look up the function in `fvars`. If `fvars` is empty, using the caller's context. `mapping` should be a tuple of paired regular expressions with function name substitutions. `handle` will import modules as necessary. """ for url, ofno in utils.group(mapping, 2): if isinstance(ofno, tuple): ofn, fna = ofno[0], list(ofno[1:]) else: ofn, fna = ofno, [] fn, result = utils.re_subm('^' + url + '$', ofn, web.ctx.path) if result: # it's a match if fn.split(' ', 1)[0] == "redirect": url = fn.split(' ', 1)[1] if web.ctx.method == "GET": x = web.ctx.env.get('QUERY_STRING', '') if x: url += '?' + x return http.redirect(url) elif '.' in fn: x = fn.split('.') mod, cls = '.'.join(x[:-1]), x[-1] mod = __import__(mod, globals(), locals(), [""]) cls = getattr(mod, cls) else: cls = fn mod = fvars if isinstance(mod, types.ModuleType): mod = vars(mod) try: cls = mod[cls] except KeyError: return web.notfound() meth = web.ctx.method if meth == "HEAD": if not hasattr(cls, meth): meth = "GET" if not hasattr(cls, meth): return nomethod(cls) tocall = getattr(cls(), meth) args = list(result.groups()) for d in re.findall(r'\\(\d+)', ofn): args.pop(int(d) - 1) return tocall(*([urllib.unquote(x) for x in args] + fna)) return web.notfound()
def handle(mapping, fvars=None): """ Call the appropriate function based on the url to function mapping in `mapping`. If no module for the function is specified, look up the function in `fvars`. If `fvars` is empty, using the caller's context. `mapping` should be a tuple of paired regular expressions with function name substitutions. `handle` will import modules as necessary. """ for url, ofno in utils.group(mapping, 2): if isinstance(ofno, tuple): ofn, fna = ofno[0], list(ofno[1:]) else: ofn, fna = ofno, [] fn, result = utils.re_subm('^' + url + '$', ofn, web.ctx.path) if result: # it's a match if fn.split(' ', 1)[0] == "redirect": url = fn.split(' ', 1)[1] if web.ctx.method == "GET": x = web.ctx.env.get('QUERY_STRING', '') if x: url += '?' + x return http.redirect(url) elif '.' in fn: x = fn.split('.') mod, cls = '.'.join(x[:-1]), x[-1] mod = __import__(mod, globals(), locals(), [""]) cls = getattr(mod, cls) else: cls = fn mod = fvars if isinstance(mod, types.ModuleType): mod = vars(mod) try: cls = mod[cls] except KeyError: return web.notfound() meth = web.ctx.method if meth == "HEAD": if not hasattr(cls, meth): meth = "GET" if not hasattr(cls, meth): return nomethod(cls) tocall = getattr(cls(), meth) args = list(result.groups()) for d in re.findall(r'\\(\d+)', ofn): args.pop(int(d) - 1) return tocall(*([x and urllib.unquote(x) for x in args] + fna)) return web.notfound()
def __call__(self, hits): words = getattr(self._cl.query, 'sphinx', self._cl.query) docs = [] for match in hits.matches: docs.extend( [utils._unicode(match['@hit'][f]) for f in self._on_fields]) all_excerpts = self._cl.BuildExcerpts(docs, self._index, words, self._opts) print all_excerpts for match, excerpts in zip( hits.matches, utils.group(all_excerpts, len(self._on_fields))): for f, excerpt in zip(self._on_fields, excerpts): match['@hit'][f + self._suffix] = excerpt or match['@hit'][f]
def _match(self, mapping, value): for pat, what in utils.group(mapping, 2): if isinstance(what, application): if value.startswith(pat): f = lambda: self._delegate_sub_application(pat, what) return f, None else: continue elif isinstance(what, basestring): what, result = utils.re_subm('^' + pat + '$', what, value) else: result = utils.re_compile('^' + pat + '$').match(value) if result: # it's a match return what, [x and urllib.unquote(x) for x in result.groups()] return None, None
def summarizeTicks(allTicks, extractKey, itemsEnd, maxCount): asks = np.zeros(maxCount) for key, g in group(extractKey, allTicks): logger.debug('New group, size={size}, key={key}, start={start}' .format(size=len(g), key=key, start=g[0].date)) n = len(g) for i in range(0, n): asks[i] = g[i].ask date = datetime.datetime(*key) itemsEnd = min(n, itemsEnd) yield Summary(date=date, askMax=np.max(asks[:n]), askMin=np.min(asks[:n]), askAverage=np.average(asks[:n]), askOpen=np.mean(asks[0:itemsEnd]), askClose=np.mean(asks[n-itemsEnd:n]))
def _match(self, mapping, value): for pat, what in utils.group(mapping, 2): if isinstance(what, application): if value.startswith(pat): f = lambda: self._delegate_sub_application(pat, what) return f, None else: continue elif isinstance(what, basestring): what, result = utils.re_subm('^' + pat + '$', what, value) else: result = utils.re_compile('^' + pat + '$').match(value) if result: # it's a match return what, [x for x in result.groups()] return None, None
def profile(): try: body = group(json.loads(request.data)) interpolated_path = [] for line in body: interpolated_path.append(interpolate(line[0], line[1])) result = [] for line in interpolated_path: for point in line: alt = int(get_altitude(point[1], point[0])) result.append([*point, alt]) print(result) return make_response(json.dumps(result), 200) except: return Response(status=404)
def __sub__(self, other): # TODO subtractionw without index variables if self.index_type is not None: index = grizzly_impl.get_field(self.expr, 0) expr1 = grizzly_impl.get_field(self.expr, 1) else: expr1 = self.expr if other.index_type is not None: index2 = grizzly_impl.get_field(other.expr, 0) expr2 = grizzly_impl.get_field(other.expr, 1) else: expr2 = other.expr index_expr = LazyOpResult(index, self.index_type, 0) sub_expr = SeriesWeld( grizzly_impl.element_wise_op(expr1, expr2, "-", self.weld_type), self.weld_type, self.df, self.column_name) index_sub_expr = utils.group([index_expr, sub_expr]) return SeriesWeld(index_sub_expr.expr, self.weld_type, self.df, self.column_name, self.index_type, self.index_name)
def evaluate(self, verbose=True, passes=None): """Summary Returns: TYPE: Description """ i = 0 exprs = [] for column_name in self.column_names: if len(self.column_names) > 1: index = "1.$%d" % i else: index = "1" expr = self.get_column(column_name, self.column_types[i], index, verbose=verbose) i += 1 exprs.append(expr) i = 0 for column_name in self.grouping_column_name: if len(self.grouping_column_name) > 1: index = "0.$%d" % i else: index = "0" expr = self.get_column(column_name, self.grouping_column_types[i], index, verbose=verbose) exprs.append(expr) i += 1 result = utils.group(exprs).evaluate(verbose=verbose, passes=passes) df = pd.DataFrame(columns=[]) all_columns = self.column_names + self.grouping_column_name for i, column_name in enumerate(all_columns): df[column_name] = result[i] return dataframeweld.DataFrameWeld(df)
def std(self): """Standard deviation Note that is by default normalizd by n - 1 # TODO, what does pandas do for multiple grouping columns? # Currently we are just going to use one grouping column """ std_expr = grizzly_impl.groupby_std([self.column], [self.column_type], self.grouping_columns, self.grouping_column_types) unzipped_columns = grizzly_impl.unzip_columns( std_expr, self.grouping_column_types + [WeldDouble()], ) index_expr = LazyOpResult(grizzly_impl.get_field(unzipped_columns, 0), self.grouping_column_types[0], 1) column_expr = LazyOpResult(grizzly_impl.get_field(unzipped_columns, 1), self.grouping_column_types[0], 1) group_expr = utils.group([index_expr, column_expr]) return SeriesWeld(group_expr.expr, WeldDouble(), index_type=self.grouping_column_types[0], index_name=self.grouping_column_names[0])
def parse(self, node): """ parse a layer element """ from utils import group from itertools import product, imap from struct import unpack import array self.set_properties(node) data = None next_gid = None data_node = node.find('data') encoding = data_node.get("encoding", None) if encoding == "base64": from base64 import decodestring data = decodestring(data_node.text.strip()) elif encoding == "csv": next_gid = imap( int, "".join(line.strip() for line in data_node.text.strip()).split(",")) elif encoding: msg = "TMX encoding type: {0} is not supported." raise Exception, msg.format(encoding) compression = data_node.get("compression", None) if compression == "gzip": from StringIO import StringIO import gzip fh = gzip.GzipFile(fileobj=StringIO(data)) data = fh.read() fh.close() elif compression == "zlib": import zlib data = zlib.decompress(data) elif compression: msg = "TMX compression type: {0} is not supported." raise Exception, msg.format(str(attr["compression"])) # if data is None, then it was not decoded or decompressed, so # we assume here that it is going to be a bunch of tile elements # TODO: this will probably raise an exception if there are no tiles if encoding == next_gid is None: def get_children(parent): for child in parent.findall('tile'): yield int(child.get('gid')) next_gid = get_children(data_node) elif data: # data is a list of gids. cast as 32-bit ints to format properly # create iterator to efficiently parse data next_gid = imap(lambda i: unpack("<L", "".join(i))[0], group(data, 4)) # using bytes here limits the layer to 256 unique tiles # may be a limitation for very detailed maps, but most maps are not # so detailed. [self.data.append(array.array("H")) for i in xrange(self.height)] for (y, x) in product(xrange(self.height), xrange(self.width)): self.data[y].append( self.parent.register_gid(*decode_gid(next(next_gid))))
def init_mapping(self, mapping): self.mapping = list(utils.group(mapping, 2))
def parse(self, node): """ parse a layer element """ from utils import group from itertools import product, imap from struct import unpack import array self.set_properties(node) data = None next_gid = None data_node = node.find('data') encoding = data_node.get("encoding", None) if encoding == "base64": from base64 import decodestring data = decodestring(data_node.text.strip()) elif encoding == "csv": next_gid = imap(int, "".join( line.strip() for line in data_node.text.strip() ).split(",")) elif encoding: msg = "TMX encoding type: {0} is not supported." raise Exception, msg.format(encoding) compression = data_node.get("compression", None) if compression == "gzip": from StringIO import StringIO import gzip fh = gzip.GzipFile(fileobj=StringIO(data)) data = fh.read() fh.close() elif compression == "zlib": import zlib data = zlib.decompress(data) elif compression: msg = "TMX compression type: {0} is not supported." raise Exception, msg.format(str(attr["compression"])) # if data is None, then it was not decoded or decompressed, so # we assume here that it is going to be a bunch of tile elements # TODO: this will probably raise an exception if there are no tiles if encoding == next_gid is None: def get_children(parent): for child in parent.findall('tile'): yield int(child.get('gid')) next_gid = get_children(data_node) elif data: # data is a list of gids. cast as 32-bit ints to format properly # create iterator to efficiently parse data next_gid = imap(lambda i: unpack("<L", "".join(i))[0], group(data, 4)) # using bytes here limits the layer to 256 unique tiles # may be a limitation for very detailed maps, but most maps are not # so detailed. [self.data.append(array.array("H")) for i in xrange(self.height)] for (y, x) in product(xrange(self.height), xrange(self.width)): self.data[y].append(self.parent.register_gid(*decode_gid(next(next_gid))))
def test(): import sys verbose = '-v' in sys.argv def assertEqual(a, b): if a == b: if verbose: sys.stderr.write('.') sys.stderr.flush() else: assert a == b, "\nexpected: %s\ngot: %s" % (repr(b), repr(a)) from utils import storage, group t = Template tests = [ lambda: t('1')(), '1\n', lambda: t('$def with ()\n1')(), '1\n', lambda: t('$def with (a)\n$a')(1), '1\n', lambda: t('$def with (a=0)\n$a')(1), '1\n', lambda: t('$def with (a=0)\n$a')(a=1), '1\n', lambda: t('$if 1: 1')(), '1\n', lambda: t('$if 1:\n 1')(), '1\n', lambda: t('$if 0: 0\n$elif 1: 1')(), '1\n', lambda: t('$if 0: 0\n$elif None: 0\n$else: 1')(), '1\n', lambda: t('$if (0 < 1) and (1 < 2): 1')(), '1\n', lambda: t('$for x in [1, 2, 3]: $x')(), '1\n2\n3\n', lambda: t('$for x in []: 0\n$else: 1')(), '1\n', lambda: t('$def with (a)\n$while a and a.pop(): 1')([1, 2, 3]), '1\n1\n1\n', lambda: t('$while 0: 0\n$else: 1')(), '1\n', lambda: t('$ a = 1\n$a')(), '1\n', lambda: t('$# 0')(), '', lambda: t('$def with (d)\n$for k, v in d.iteritems(): $k')({ 1: 1 }), '1\n', lambda: t('$def with (a)\n$(a)')(1), '1\n', lambda: t('$def with (a)\n$a')(1), '1\n', lambda: t('$def with (a)\n$a.b')(storage(b=1)), '1\n', lambda: t('$def with (a)\n$a[0]')([1]), '1\n', lambda: t('${0 or 1}')(), '1\n', lambda: t('$ a = [1]\n$a[0]')(), '1\n', lambda: t('$ a = {1: 1}\n$a.keys()[0]')(), '1\n', lambda: t('$ a = []\n$if not a: 1')(), '1\n', lambda: t('$ a = {}\n$if not a: 1')(), '1\n', lambda: t('$ a = -1\n$a')(), '-1\n', lambda: t('$ a = "1"\n$a')(), '1\n', lambda: t('$if 1 is 1: 1')(), '1\n', lambda: t('$if not 0: 1')(), '1\n', lambda: t('$if 1:\n $if 1: 1')(), '1\n', lambda: t('$ a = 1\n$a')(), '1\n', lambda: t('$ a = 1.\n$a')(), '1.0\n', lambda: t('$({1: 1}.keys()[0])')(), '1\n', ] for func, value in group(tests, 2): assertEqual(func(), value) j = Template("$var foo: bar")() assertEqual(str(j), '') assertEqual(j.foo, 'bar\n') if verbose: sys.stderr.write('\n')
def group_by_mean(result): # group by training seed and reduce by mean result = utils.group(result, 5) result = utils.reduce_by_mean(result) return result
def group_by_max(result): result = utils.group(result, 5) result = utils.reduce_by_max_val(result) return result
#calculator for loan maturity #imports from datetime import datetime import utils ##### interests = utils.loaddata(utils.interests) txs = utils.loaddata(utils.txs) list_txs = utils.group(txs) list_interests = utils.group(interests) del interests, txs list_txs = utils.reindex(list_txs) list_interests = utils.reindex(list_interests) list_txs[7] = list_txs[7].append(list_txs[8]) del list_txs[8] list_txs.insert(6, list_txs[3]) del list_txs[3] symbols = [] comp_interest = [] apys = [] for i in list_interests: symbols.append(i['SYMBOL'][0]) comp_interest.append(i['COMP'].mean()) apys.append(i['APY'].mean()) ####
def add_mapping(self, map): self.mapping.extend(utils.group(map, 2))
def get_bureau(path, num_rows=None): """Preprocess and extract features from bureau and bureau balance. Get bureau balance features grouped by SK_ID_BUREAU and append to bureau data. After that, it performs aggregations for each customer (unique SK_ID_CURR) and return a DataFrame Arguments: path: Path to the folder where files are saved (string). num_rows: Number of rows to read; None reads all rows (int, default: None). Returns: df: DataFrame with processed data. """ bureau = pd.read_csv(os.path.join(path, 'bureau.csv'), nrows= num_rows) # Credit duration and credit/account end date difference bureau['CREDIT_DURATION'] = -bureau['DAYS_CREDIT'] + bureau['DAYS_CREDIT_ENDDATE'] bureau['ENDDATE_DIF'] = bureau['DAYS_CREDIT_ENDDATE'] - bureau['DAYS_ENDDATE_FACT'] # Credit to debt ratio and difference bureau['DEBT_PERCENTAGE'] = bureau['AMT_CREDIT_SUM'] / bureau['AMT_CREDIT_SUM_DEBT'] bureau['DEBT_CREDIT_DIFF'] = bureau['AMT_CREDIT_SUM'] - bureau['AMT_CREDIT_SUM_DEBT'] bureau['CREDIT_TO_ANNUITY_RATIO'] = bureau['AMT_CREDIT_SUM'] / bureau['AMT_ANNUITY'] # One-hot encoder bureau, _ = utils.one_hot_encoder(bureau, nan_as_category= False) # Join bureau balance features bureau = bureau.merge(get_bureau_balance(path, num_rows), how='left', on='SK_ID_BUREAU') # Flag months with late payments (days past due) bureau['STATUS_12345'] = 0 for i in range(1,6): bureau['STATUS_12345'] += bureau['STATUS_{}'.format(i)] # Aggregate by number of months in balance and merge with bureau (loan length agg) features = ['AMT_CREDIT_MAX_OVERDUE', 'AMT_CREDIT_SUM_OVERDUE', 'AMT_CREDIT_SUM', 'AMT_CREDIT_SUM_DEBT', 'DEBT_PERCENTAGE', 'DEBT_CREDIT_DIFF', 'STATUS_0', 'STATUS_12345'] agg_length = bureau.groupby('MONTHS_BALANCE_SIZE')[features].mean().reset_index() agg_length.rename({feat: 'LL_' + feat for feat in features}, axis=1, inplace=True) bureau = bureau.merge(agg_length, how='left', on='MONTHS_BALANCE_SIZE') del agg_length gc.collect() # General loans aggregations agg_bureau = utils.group(bureau, 'BUREAU_', config.BUREAU_AGG) # Active and closed loans aggregations active = bureau[bureau['CREDIT_ACTIVE_Active'] == 1] agg_bureau = utils.group_and_merge(active,agg_bureau,'BUREAU_ACTIVE_',config.BUREAU_ACTIVE_AGG) closed = bureau[bureau['CREDIT_ACTIVE_Closed'] == 1] agg_bureau = utils.group_and_merge(closed,agg_bureau,'BUREAU_CLOSED_',config.BUREAU_CLOSED_AGG) del active, closed gc.collect() # Aggregations for the main loan types for credit_type in ['Consumer credit', 'Credit card', 'Mortgage', 'Car loan', 'Microloan']: type_df = bureau[bureau['CREDIT_TYPE_' + credit_type] == 1] prefix = 'BUREAU_' + credit_type.split(' ')[0].upper() + '_' agg_bureau = utils.group_and_merge(type_df, agg_bureau, prefix, config.BUREAU_LOAN_TYPE_AGG) del type_df gc.collect() # Time based aggregations: last x months for time_frame in [6, 12, 24, 36]: prefix = "BUREAU_LAST{}M_".format(time_frame) time_frame_df = bureau[bureau['DAYS_CREDIT'] >= -30*time_frame] agg_bureau = utils.group_and_merge(time_frame_df,agg_bureau,prefix,config.BUREAU_TIME_AGG) del time_frame_df gc.collect() # Last loan max overdue sort_bureau = bureau.sort_values(by=['DAYS_CREDIT']) gr = sort_bureau.groupby('SK_ID_CURR')['AMT_CREDIT_MAX_OVERDUE'].last().reset_index() gr.rename({'AMT_CREDIT_MAX_OVERDUE': 'BUREAU_LAST_LOAN_MAX_OVERDUE'}, inplace=True) agg_bureau = agg_bureau.merge(gr, on='SK_ID_CURR', how='left') # Ratios: total debt/total credit and active loans debt/ active loans credit agg_bureau['BUREAU_DEBT_OVER_CREDIT'] = \ agg_bureau['BUREAU_AMT_CREDIT_SUM_DEBT_SUM']/agg_bureau['BUREAU_AMT_CREDIT_SUM_SUM'] agg_bureau['BUREAU_ACTIVE_DEBT_OVER_CREDIT'] = \ agg_bureau['BUREAU_ACTIVE_AMT_CREDIT_SUM_DEBT_SUM']/agg_bureau['BUREAU_ACTIVE_AMT_CREDIT_SUM_SUM'] return agg_bureau
def mtime_beat(): '''每次任务只跑一年的''' y_list = [] y = get_year() + 1 # 要抓取的年份 debug('Fetch Year: {} starting...'.format(y)) instance = fetch(y, 1) page = get_movie_pages(instance) if page is None: warn('Movie"page has not fetched') # 执行间隔自适应 if scheduler.get_interval < TASK_BEAT * 7: scheduler.change_interval(incr=True) return ids = get_movie_ids(instance) if ids is None: # 间隔自适应也不能太大 warn('Movie has not fetched') if scheduler.get_interval < TASK_BEAT * 7: scheduler.change_interval(incr=True) return # 当任务继续能执行的时候,回到默认的间隔 if scheduler.get_interval > TASK_BEAT: debug('Interval back to default') scheduler.change_interval(TASK_BEAT) y_list.extend(ids) if not y_list: # 本年没有电影 debug('Year: {} has not movie'.format(y)) YearFinished(year=y).save() sleep2() return mtime_beat() if page > 1: p = 2 while p <= page: instance = fetch(y, p) debug('Fetch Year:{} Page:{}'.format(y, p)) ids = get_movie_ids(instance) if ids is None: # 间隔自适应也不能太大 if scheduler.get_interval < TASK_BEAT * 7: scheduler.change_interval(incr=True) # 出现需要验证码 手动输入或者等待一段时间后重试,直到能正常使用 sleep2(VERIFY_INTERVAL) continue ids = [] y_list.extend(ids) p += 1 sleep2() obj = IdFinished.objects(year=y).first() if obj is not None: has_finished = obj.ids else: has_finished = [] to_process = get_unfinished(has_finished, y_list) # 给相应队列添加任务 for payload in group(to_process, TASK_BEAT_NUM): for task in ['Fullcredits', 'Movie', 'Comment', 'Character', 'MicroComment', 'Scenes', 'Awards', 'Plot', 'Details']: debug('Push payload: {} to {} Queue'.format(payload, task)) try: Message(year=y, task=task, payload=payload).save() # Hack一下 #Message.objects.get_or_create(year=y, task=task, payload=payload) except NotUniqueError: debug('Duplicate insert: [{}], payload: {}'.format(task, payload)) # 当前年份数据已经入MQ YearFinished(year=y).save() debug('Year: {} done'.format(y))
end = args["end_val"] delta = args["delta"] * 1.0 if start == end: return 1 if delta > 0 and start > end: return 0 elif delta < 0 and end > start: return 0 elif start <= end: return iceil((end + 1 - start) / abs(delta)) else: return iceil((start + 1 - end) / abs(delta)) TESTS_GROUPS = group( flatten([cls().build_tests() for cls in Gbs3TestScript.__subclasses__()]), 128 ) TEST_DIR = os.path.dirname(os.path.dirname(__file__)) THIS_TEST_DIR = os.path.dirname(__file__) class AutoTestCaseGbs3(unittest.TestCase, AutoTestCase): def setUp(self): copy_file( THIS_TEST_DIR + "/Biblioteca.gbs", TEST_DIR + "/examples/Biblioteca.gbs" ) def gobstones_parameters(self): return "--language gobstones"
def cmd_setup(args): """Setup environment.""" if not args.dirs: args.kernel = False args.rdma_core = False args.iproute = False if args.installs: print(""" This setup script will update your hypervisor to latest distribution packages and install docker. Please restart the hypervisor to complete the installation. """) if args.yes is False and utils.query_yes_no("Do you want to proceed?", 'no') is False: exit("Exiting ...") supported_os = { 'fedora': '26', 'ubuntu': '16', 'rhel': '8', 'redhat': '8', } # Python API stability is dissaster # module platform was deprecated https://docs.python.org/3/library/platform.html # Luckily enough, they added distro module before removing platform try: import distro distro_id = distro.id() distro_v = distro.major_version() except ModuleNotFoundError: import platform distro_id = platform.dist()[0].lower() distro_v = platform.dist()[1].split('.')[0] if distro_id not in supported_os.keys( ) or distro_v < supported_os[distro_id]: exit(""" Your hypervisor is not supported. Exiting ...""") if args.installs: setuphv = utils.get_internal_fn('scripts/') if distro_id == 'redhat': distro_id = 'rhel' setuphv += 'setup-hv.' + distro_id subprocess.check_call(setuphv) utils.init_config_file() section = utils.load_config_file() if args.dirs: for key, value in section.items(): if args.force: subprocess.call(["sudo", "rm", "-rf", value]) if os.path.exists(value): exit("Please remove " + value + " Exiting ...") if key == "kernel" and not args.kernel: continue if key == "rdma-core" and not args.rdma_core: continue if key == "iproute2" and not args.iproute: continue if key == 'os': continue print("Prepare " + key) subprocess.call(["sudo", "mkdir", "-p", value]) subprocess.call([ "sudo", "chown", "-R", utils.username() + ":" + utils.group(), value ]) if key == "src" or key == "logs" or key == "ccache": continue if key == "kernel": key = "linux" p = subprocess.Popen([ "git", "clone", "ssh://" + utils.username() + "@l-gerrit.mtl.labs.mlnx:29418/upstream/" + key, "." ], cwd=value) p.wait() p = subprocess.Popen([ "scp", "-p", "-P", "29418", utils.username() + "@l-gerrit.mtl.labs.mlnx:hooks/commit-msg", ".git/hooks/" ], cwd=value) p.wait() if key == "linux": shutil.copy( os.path.join(os.path.dirname(__file__), "../configs/kconfig-kvm"), value + "/.config") print("Completed, PLEASE RESTART server")