def cmd_setup_slave(args):
''' Setup proper slave environment and connect to master server '''
if not args.export:
cmd = ["mkt", "setup", "--no-dirs"]
if args.yes:
cmd.append("-y")
if args.force:
cmd.append("-f")
subprocess.call(cmd)
utils.init_config_file()
section = utils.load_config_file()
subprocess.call(["sudo", "mkdir", "-p", section['src']])
subprocess.call(["sudo", "chown", "-R", utils.username() + ":" + utils.group(), section['src']])
subprocess.call(["sudo", "mkdir", "-p", section['logs']])
subprocess.call(["sudo", "chown", "-R", utils.username() + ":" + utils.group(), section['logs']])
with tempfile.NamedTemporaryFile("w") as f:
f.write(open("/etc/fstab").read())
export = args.hostname[0] + ":" + \
section['src'][:-1] + "\t" + section['src'][:-1] + \
"\t" + "nfs" + "\t" + "ro,nolock 0 0" + "\n" + \
args.hostname[0] + ":" + \
section['logs'][:-1] + "\t" + section['logs'][:-1] + \
"\t" + "nfs" + "\t" + "rw,nolock 0 0" + "\n"
f.write(export)
f.flush()
subprocess.call(["sudo", "cp", f.name, "/etc/fstab"])
subprocess.call(["sudo", "mount", "-a", "-t", "nfs", "-o", "remount"])
def group_by_best_model(result):
# select best model by max val acc
result = utils.group(result, 5)
result = utils.reduce_by_max_val(result)
result = utils.group(result, 4, keepdim=True)
result = utils.reduce_by_max_val(result, dim=4, dim_name="model")
return result
def test():
import sys
verbose = '-v' in sys.argv
def assertEqual(a, b):
if a == b:
if verbose:
sys.stderr.write('.')
sys.stderr.flush()
else:
assert a == b, "\nexpected: %s\ngot: %s" % (repr(b), repr(a))
from utils import storage, group
t = Template
tests = [
lambda: t('1')(), '1\n',
lambda: t('$def with ()\n1')(), '1\n',
lambda: t('$def with (a)\n$a')(1), '1\n',
lambda: t('$def with (a=0)\n$a')(1), '1\n',
lambda: t('$def with (a=0)\n$a')(a=1), '1\n',
lambda: t('$if 1: 1')(), '1\n',
lambda: t('$if 1:\n 1')(), '1\n',
lambda: t('$if 0: 0\n$elif 1: 1')(), '1\n',
lambda: t('$if 0: 0\n$elif None: 0\n$else: 1')(), '1\n',
lambda: t('$if (0 < 1) and (1 < 2): 1')(), '1\n',
lambda: t('$for x in [1, 2, 3]: $x')(), '1\n2\n3\n',
lambda: t('$for x in []: 0\n$else: 1')(), '1\n',
lambda: t('$def with (a)\n$while a and a.pop(): 1')([1, 2, 3]), '1\n1\n1\n',
lambda: t('$while 0: 0\n$else: 1')(), '1\n',
lambda: t('$ a = 1\n$a')(), '1\n',
lambda: t('$# 0')(), '',
lambda: t('$def with (d)\n$for k, v in d.iteritems(): $k')({1: 1}), '1\n',
lambda: t('$def with (a)\n$(a)')(1), '1\n',
lambda: t('$def with (a)\n$a')(1), '1\n',
lambda: t('$def with (a)\n$a.b')(storage(b=1)), '1\n',
lambda: t('$def with (a)\n$a[0]')([1]), '1\n',
lambda: t('${0 or 1}')(), '1\n',
lambda: t('$ a = [1]\n$a[0]')(), '1\n',
lambda: t('$ a = {1: 1}\n$a.keys()[0]')(), '1\n',
lambda: t('$ a = []\n$if not a: 1')(), '1\n',
lambda: t('$ a = {}\n$if not a: 1')(), '1\n',
lambda: t('$ a = -1\n$a')(), '-1\n',
lambda: t('$ a = "1"\n$a')(), '1\n',
lambda: t('$if 1 is 1: 1')(), '1\n',
lambda: t('$if not 0: 1')(), '1\n',
lambda: t('$if 1:\n $if 1: 1')(), '1\n',
lambda: t('$ a = 1\n$a')(), '1\n',
lambda: t('$ a = 1.\n$a')(), '1.0\n',
lambda: t('$({1: 1}.keys()[0])')(), '1\n',
lambda: t('$for x in [1, 2, 3]:\n\t$x') (), ' 1\n 2\n 3\n',
lambda: t('$def with (a)\n$:a')(1), '1\n',
]
for func, value in group(tests, 2):
assertEqual(func(), value)
j = Template("$var foo: bar")()
assertEqual(str(j), '')
assertEqual(j.foo, 'bar\n')
if verbose: sys.stderr.write('\n')
def bin_points(x_pnts, y_pnts, min_bucket, split_bucket): # adaptive min_bucket and split_bucket adaptive_min_bucket = isinstance(min_bucket, str) and \ min_bucket == 'adaptive' adaptive_split_bucket = isinstance(split_bucket, str) and \ split_bucket == 'adaptive' if adaptive_min_bucket or adaptive_split_bucket: contrast_range = x_pnts[[0, -1]] # or if we want to trim edges: # drop_elements = int(np.ceil(len(x_pnts) * 0.08)) # contrast_range = x_pnts[drop_elements:-drop_elements][[0, -1]] contrast_range = np.diff(contrast_range)[0] if adaptive_min_bucket: min_bucket = contrast_range / 20. if adaptive_split_bucket: split_bucket = contrast_range / 10. # look for buckets x_buckets = group(np.diff(x_pnts) <= min_bucket) n_pnts_in_bucket = (np.diff(x_buckets, axis=-1) + 1).ravel() good_buckets = n_pnts_in_bucket >= (3 - 1) # -1 because of diff if x_buckets.shape[0] > 0 and np.any(good_buckets): x_buckets = x_buckets[good_buckets, :] # turn buckets to slices, get mean and sem x_buckets[:, 1] += 2 # +1 because of python slicing # another +1 because of diff slices = [slice(l[0], l[1]) for l in x_buckets] # test each slice for contrast range and split if needed add_slices = list() ii = 0 while ii < len(slices): slc = slices[ii] pnts = x_pnts[slc] l, h = pnts[[0, -1]] rng = h - l start = slc.start if rng > split_bucket: slices.pop(ii) n_full_splits = int(np.floor(rng / split_bucket)) last_l = l last_ind = 0 for splt in range(n_full_splits): this_high = last_l + split_bucket this_ind = np.where(pnts >= this_high)[0][0] add_slices.append(slice(start + last_ind, start + this_ind + 1)) last_l = this_high last_ind = this_ind + 1 # last one - to the end if start + last_ind < slc.stop: add_slices.append(slice(start + last_ind, slc.stop)) else: ii += 1 slices.extend(add_slices) return slices
def find_elements(diff_array):
diff_array.flags.writeable = True
diff_array[22:92,160:320] = zeros((70,160))
contours,hierarchy = cv2.findContours(diff_array, 1, 2)
bird = None
raw_pipes = []
for cnt in contours:
area = cv2.contourArea(cnt)
rect = cv2.boundingRect(cnt)
if 1400 < area < 1700 and 90 < rect[0] < 170:
bird = rect
elif area > 1500:
raw_pipes.append(rect)
pipes = []
for pipe in group(sorted(raw_pipes, key=lambda x:x[0]), 2):
x = min(pipe[0][0], pipe[1][0])
y_values = [pipe[0][1], pipe[1][1]]
min_y_index, min_y_value = min(enumerate(y_values), key=operator.itemgetter(1))
y = min_y_value + [pipe[0][3], pipe[1][3]][min_y_index]
size_x = min(pipe[0][2], pipe[1][2])
size_y = y_values[min_y_index-1] - y_values[min_y_index] - y + min_y_value
p = [x, y, size_x, size_y]
pipes.append(p)
return bird, pipes
def _match(self, mapping, value):
for pat, what in utils.group(mapping, 2):
rx = utils.re_compile('^' + pat + '$')
result = rx.match(value)
if result:
return what, [x and urllib.unquote(x) for x in result.groups()]
return None, None
def _match(self, mapping, value):
for pat, what in utils.group(mapping, 2):
rx = utils.re_compile("^" + pat + "$")
result = rx.match(value)
if result:
return what, [x and urllib.unquote(x) for x in result.groups()]
return None, None
def bin_points(x_pnts, y_pnts, min_bucket, split_bucket):
# adaptive min_bucket and split_bucket
adaptive_min_bucket = isinstance(min_bucket, str) and \
min_bucket == 'adaptive'
adaptive_split_bucket = isinstance(split_bucket, str) and \
split_bucket == 'adaptive'
if adaptive_min_bucket or adaptive_split_bucket:
contrast_range = x_pnts[[0, -1]]
# or if we want to trim edges:
# drop_elements = int(np.ceil(len(x_pnts) * 0.08))
# contrast_range = x_pnts[drop_elements:-drop_elements][[0, -1]]
contrast_range = np.diff(contrast_range)[0]
if adaptive_min_bucket:
min_bucket = contrast_range / 20.
if adaptive_split_bucket:
split_bucket = contrast_range / 10.
# look for buckets
x_buckets = group(np.diff(x_pnts) <= min_bucket)
n_pnts_in_bucket = (np.diff(x_buckets, axis=-1) + 1).ravel()
good_buckets = n_pnts_in_bucket >= (3 - 1) # -1 because of diff
if x_buckets.shape[0] > 0 and np.any(good_buckets):
x_buckets = x_buckets[good_buckets, :]
# turn buckets to slices, get mean and sem
x_buckets[:, 1] += 2 # +1 because of python slicing
# another +1 because of diff
slices = [slice(l[0], l[1]) for l in x_buckets]
# test each slice for contrast range and split if needed
add_slices = list()
ii = 0
while ii < len(slices):
slc = slices[ii]
pnts = x_pnts[slc]
l, h = pnts[[0, -1]]
rng = h - l
start = slc.start
if rng > split_bucket:
slices.pop(ii)
n_full_splits = int(np.floor(rng / split_bucket))
last_l = l
last_ind = 0
for splt in range(n_full_splits):
this_high = last_l + split_bucket
this_ind = np.where(pnts >= this_high)[0][0]
add_slices.append(
slice(start + last_ind, start + this_ind + 1))
last_l = this_high
last_ind = this_ind + 1
# last one - to the end
if start + last_ind < slc.stop:
add_slices.append(slice(start + last_ind, slc.stop))
else:
ii += 1
slices.extend(add_slices)
return slices
def view_raw(args):
data = read_raw(args.view_count)
for name, _, cls, bbox in data:
print(name, cls, bbox)
for d in group(data, 36):
# display raw images
_, images, _, _ = zip(*d)
show_image(merge_images_with_border([cv2.resize(i, (150, 150)) for i in images]))
def _match(self, mapping, value):
for pat, what in utils.group(mapping, 2):
if isinstance(what, basestring):
what, result = utils.re_subm("^" + pat + "$", what, web.ctx.path)
else:
result = utils.re_compile("^" + pat + "$").match(web.ctx.path)
if result: # it's a match
return what, [x and urllib.unquote(x) for x in result.groups()]
return None, None
def get_vector_carno(vector):
vector = vector[::-1]
arr = [0, 1, 3, 2, 4, 5, 7, 6, 12, 13, 15, 14, 8, 9, 11, 10]
groups = group(arr, 4)
lst = [['', 'cd', 'cD', 'CD', 'Cd']]
rows = ['ab', 'aB', 'AB', 'Ab']
for i, grp in enumerate(groups):
lst += [[rows[i]] + [vector[x] for x in grp]]
table_view(lst, free_space_right=1)
def _match(self, mapping, value):
for pat, what in utils.group(mapping, 2):
if isinstance(what, basestring):
what, result = utils.re_subm('^' + pat + '$', what, value)
else:
result = utils.re_compile('^' + pat + '$').match(value)
if result: # it's a match
return what, [x for x in result.groups()]
return None, None
def _match(self, mapping, value):
for pat, what in utils.group(mapping, 2):
if isinstance(what, basestring):
what, result = utils.re_subm('^' + pat + '$', what, value)
else:
result = utils.re_compile('^' + pat + '$').match(value)
if result: # it's a match
return what, [x for x in result.groups()]
return None, None
def create_listsoflists():
global tag_list
tag_list = connection.list_tags()
name_list = list()
for tag in tag_list:
name_list.append(tag.name)
tag_grouped = utils.group(name_list, 3)
return tag_grouped
def handle(self):
for dir, what in utils.group(self.mapping, 2):
if web.ctx.path.startswith(dir + '/'):
# change paths to make path relative to dir
web.ctx.home += dir
web.ctx.homepath += dir
web.ctx.path = web.ctx.path[len(dir):]
web.ctx.fullpath = web.ctx.fullpath[len(dir):]
return self._delegate(what, self.fvars)
raise NotFound
def get_installment_payments(path, num_rows=None):
"""Preprocess and extract features from installments_payments.
Arguments:
path: Path to the folder where files are saved (string).
num_rows: Number of rows to read; None reads all rows (int, default: None).
Returns:
df: DataFrame with processed data.
"""
pay = pd.read_csv(os.path.join(path, 'installments_payments.csv'), nrows=num_rows)
# Group payments and get Payment difference
pay = utils.do_sum(pay, ['SK_ID_PREV', 'NUM_INSTALMENT_NUMBER'], 'AMT_PAYMENT', 'AMT_PAYMENT_GROUPED')
pay['PAYMENT_DIFFERENCE'] = pay['AMT_INSTALMENT'] - pay['AMT_PAYMENT_GROUPED']
pay['PAYMENT_RATIO'] = pay['AMT_INSTALMENT'] / pay['AMT_PAYMENT_GROUPED']
pay['PAID_OVER_AMOUNT'] = pay['AMT_PAYMENT'] - pay['AMT_INSTALMENT']
pay['PAID_OVER'] = (pay['PAID_OVER_AMOUNT'] > 0).astype(int)
# Payment Entry: Days past due and Days before due
pay['DPD'] = pay['DAYS_ENTRY_PAYMENT'] - pay['DAYS_INSTALMENT']
pay['DPD'] = pay['DPD'].apply(lambda x: 0 if x <= 0 else x)
pay['DBD'] = pay['DAYS_INSTALMENT'] - pay['DAYS_ENTRY_PAYMENT']
pay['DBD'] = pay['DBD'].apply(lambda x: 0 if x <= 0 else x)
# Flag late payment
pay['LATE_PAYMENT'] = pay['DBD'].apply(lambda x: 1 if x > 0 else 0)
# Percentage of payments that were late
pay['INSTALMENT_PAYMENT_RATIO'] = pay['AMT_PAYMENT'] / pay['AMT_INSTALMENT']
pay['LATE_PAYMENT_RATIO'] = pay.apply(lambda x: x['INSTALMENT_PAYMENT_RATIO'] if x['LATE_PAYMENT'] == 1 else 0, axis=1)
# Flag late payments that have a significant amount
pay['SIGNIFICANT_LATE_PAYMENT'] = pay['LATE_PAYMENT_RATIO'].apply(lambda x: 1 if x > 0.05 else 0)
# Flag k threshold late payments
pay['DPD_7'] = pay['DPD'].apply(lambda x: 1 if x > 7 else 0)
pay['DPD_15'] = pay['DPD'].apply(lambda x: 1 if x > 15 else 0)
# Aggregations by SK_ID_CURR
pay_agg = utils.group(pay, 'INS_', config.INSTALLMENTS_AGG)
# Installments in the last x months
for months in [18, 36]:
recent_prev_id = pay[pay['DAYS_INSTALMENT'] >= -30*months]['SK_ID_PREV'].unique()
pay_recent = pay[pay['SK_ID_PREV'].isin(recent_prev_id)]
prefix = 'INS_{}M_'.format(months)
pay_agg = utils.group_and_merge(pay_recent, pay_agg, prefix, config.INSTALLMENTS_TIME_AGG)
# Last x periods trend features
group_features = ['SK_ID_CURR', 'SK_ID_PREV', 'DPD', 'LATE_PAYMENT',
'PAID_OVER_AMOUNT', 'PAID_OVER', 'DAYS_INSTALMENT']
group = pay[group_features].groupby('SK_ID_CURR')
func = partial(_trend_in_last_k_instalment_features, periods=[12, 24, 60, 120])
g = utils.parallel_apply(group, func, index_name='SK_ID_CURR', chunk_size=10000).reset_index()
pay_agg = pay_agg.merge(g, on='SK_ID_CURR', how='left')
# Last loan features
g = utils.parallel_apply(group, _installments_last_loan_features, index_name='SK_ID_CURR', chunk_size=10000).reset_index()
pay_agg = pay_agg.merge(g, on='SK_ID_CURR', how='left')
return pay_agg
def handle(self):
for dir, what in utils.group(self.mapping, 2):
if web.ctx.path.startswith(dir + '/'):
# change paths to make path relative to dir
web.ctx.home += dir
web.ctx.homepath += dir
web.ctx.path = web.ctx.path[len(dir):]
web.ctx.fullpath = web.ctx.fullpath[len(dir):]
return self._delegate(what, self.fvars)
raise NotFound
async def set_lang(self, chat_id, command, args):
lang = args.strip() if args else None
if lang:
await self.save_user_settings(chat_id, {'lang': LANG.get(lang)})
await bot.sendMessage(chat_id, 'language saved', reply_markup=ReplyKeyboardRemove())
else:
markup = ReplyKeyboardMarkup(
keyboard=group(['/lang' + x for x in LANG.keys()], 2),
one_time_keyboard=True
)
await bot.sendMessage(chat_id, 'set language', reply_markup=markup)
async def set_cc(self, chat_id, command, args):
cc = args.strip() if args else None
if cc:
await self.save_user_settings(chat_id, {'cc': CC.get(cc)})
await bot.sendMessage(chat_id, 'region saved', reply_markup=ReplyKeyboardRemove())
else:
markup = ReplyKeyboardMarkup(
keyboard=group(['/cc' + x for x in CC.keys()], 3),
one_time_keyboard=True
)
await bot.sendMessage(chat_id, 'set region', reply_markup=markup)
def get_pos_cash(path, num_rows= None):
"""Preprocess and extract features from POS_CASH_balance file.
Arguments:
path: Path to the folder where files are saved (string).
num_rows: Number of rows to read; None reads all rows (int, default: None).
Returns:
df: DataFrame with processed data.
"""
pos = pd.read_csv(os.path.join(path, 'POS_CASH_balance.csv'), nrows=num_rows)
pos, categorical_cols = utils.one_hot_encoder(pos, nan_as_category=False)
# Flag months with late payment
pos['LATE_PAYMENT'] = pos['SK_DPD'].apply(lambda x: 1 if x > 0 else 0)
# Aggregate by SK_ID_CURR
categorical_agg = {key: ['mean'] for key in categorical_cols}
pos_agg = utils.group(pos, 'POS_', {**config.POS_CASH_AGG, **categorical_agg})
# Sort and group by SK_ID_PREV
sort_pos = pos.sort_values(by=['SK_ID_PREV', 'MONTHS_BALANCE'])
gp = sort_pos.groupby('SK_ID_PREV')
df = pd.DataFrame()
df['SK_ID_CURR'] = gp['SK_ID_CURR'].first()
df['MONTHS_BALANCE_MAX'] = gp['MONTHS_BALANCE'].max()
# Percentage of previous loans completed and completed before initial term
df['POS_LOAN_COMPLETED_MEAN'] = gp['NAME_CONTRACT_STATUS_Completed'].mean()
df['POS_COMPLETED_BEFORE_MEAN'] = gp['CNT_INSTALMENT'].first() - gp['CNT_INSTALMENT'].last()
df['POS_COMPLETED_BEFORE_MEAN'] = df.apply(lambda x: 1 if x['POS_COMPLETED_BEFORE_MEAN'] > 0
and x['POS_LOAN_COMPLETED_MEAN'] > 0 else 0, axis=1)
# Number of remaining installments (future installments) and percentage from total
df['POS_REMAINING_INSTALMENTS'] = gp['CNT_INSTALMENT_FUTURE'].last()
df['POS_REMAINING_INSTALMENTS_RATIO'] = gp['CNT_INSTALMENT_FUTURE'].last()/gp['CNT_INSTALMENT'].last()
# Group by SK_ID_CURR and merge
df_gp = df.groupby('SK_ID_CURR').sum().reset_index()
df_gp.drop(['MONTHS_BALANCE_MAX'], axis=1, inplace= True)
pos_agg = pd.merge(pos_agg, df_gp, on= 'SK_ID_CURR', how= 'left')
del df, gp, df_gp, sort_pos
gc.collect()
# Percentage of late payments for the 3 most recent applications
pos = utils.do_sum(pos, ['SK_ID_PREV'], 'LATE_PAYMENT', 'LATE_PAYMENT_SUM')
# Last month of each application
last_month_df = pos.groupby('SK_ID_PREV')['MONTHS_BALANCE'].idxmax()
# Most recent applications (3)
sort_pos = pos.sort_values(by=['SK_ID_PREV', 'MONTHS_BALANCE'])
gp = sort_pos.iloc[last_month_df].groupby('SK_ID_CURR').tail(3)
gp_mean = gp.groupby('SK_ID_CURR').mean().reset_index()
pos_agg = pd.merge(pos_agg, gp_mean[['SK_ID_CURR','LATE_PAYMENT_SUM']], on='SK_ID_CURR', how='left')
# Drop some useless categorical features
drop_features = [
'POS_NAME_CONTRACT_STATUS_Canceled_MEAN', 'POS_NAME_CONTRACT_STATUS_Amortized debt_MEAN',
'POS_NAME_CONTRACT_STATUS_XNA_MEAN']
pos_agg.drop(drop_features, axis=1, inplace=True)
return pos_agg
async def set_cc(self, chat_id, command, args):
cc = args.strip() if args else None
if cc:
await self.save_user_settings(chat_id, {'cc': CC.get(cc)})
await bot.sendMessage(chat_id,
'region saved',
reply_markup=ReplyKeyboardRemove())
else:
markup = ReplyKeyboardMarkup(keyboard=group(
['/cc' + x for x in CC.keys()], 3),
one_time_keyboard=True)
await bot.sendMessage(chat_id, 'set region', reply_markup=markup)
async def set_lang(self, chat_id, command, args):
lang = args.strip() if args else None
if lang:
await self.save_user_settings(chat_id, {'lang': LANG.get(lang)})
await bot.sendMessage(chat_id,
'language saved',
reply_markup=ReplyKeyboardRemove())
else:
markup = ReplyKeyboardMarkup(keyboard=group(
['/lang' + x for x in LANG.keys()], 2),
one_time_keyboard=True)
await bot.sendMessage(chat_id, 'set language', reply_markup=markup)
def __call__(self, hits):
words = getattr(self._cl.query, 'sphinx', self._cl.query)
docs = []
for match in hits.matches:
docs.extend([utils._unicode(match['@hit'][f]) for f in self._on_fields])
all_excerpts = self._cl.BuildExcerpts(docs, self._index, words, self._opts)
print all_excerpts
for match, excerpts in zip(hits.matches, utils.group(all_excerpts, len(self._on_fields))):
for f, excerpt in zip(self._on_fields, excerpts):
match['@hit'][f + self._suffix] = excerpt or match['@hit'][f]
def get_shuffle_tokens(self):
#shuffle_keys = []
#for s in string.lowercase:
# for s2 in string.lowercase:
# shuffle_keys.append(s+s2)
#for s in string.uppercase:
# for s2 in string.lowercase+string.uppercase:
# shuffle_keys.append(s+s2)
#
#return dict([(('shuffle', shard), Token()) for shard in shuffle_keys])
# return dict([(('shuffle', shard), Token()) for shard in string.lowercase+string.uppercase])
return dict([(('shuffle', tuple(g)), Token()) for g in
group(list(self.shuffle_keys), ShuffleKeysPerShard)])
def handle(mapping, fvars=None):
"""
Call the appropriate function based on the url to function mapping in `mapping`.
If no module for the function is specified, look up the function in `fvars`. If
`fvars` is empty, using the caller's context.
`mapping` should be a tuple of paired regular expressions with function name
substitutions. `handle` will import modules as necessary.
"""
for url, ofno in utils.group(mapping, 2):
if isinstance(ofno, tuple):
ofn, fna = ofno[0], list(ofno[1:])
else:
ofn, fna = ofno, []
fn, result = utils.re_subm('^' + url + '$', ofn, web.ctx.path)
if result: # it's a match
if fn.split(' ', 1)[0] == "redirect":
url = fn.split(' ', 1)[1]
if web.ctx.method == "GET":
x = web.ctx.env.get('QUERY_STRING', '')
if x:
url += '?' + x
return http.redirect(url)
elif '.' in fn:
x = fn.split('.')
mod, cls = '.'.join(x[:-1]), x[-1]
mod = __import__(mod, globals(), locals(), [""])
cls = getattr(mod, cls)
else:
cls = fn
mod = fvars
if isinstance(mod, types.ModuleType):
mod = vars(mod)
try:
cls = mod[cls]
except KeyError:
return web.notfound()
meth = web.ctx.method
if meth == "HEAD":
if not hasattr(cls, meth):
meth = "GET"
if not hasattr(cls, meth):
return nomethod(cls)
tocall = getattr(cls(), meth)
args = list(result.groups())
for d in re.findall(r'\\(\d+)', ofn):
args.pop(int(d) - 1)
return tocall(*([urllib.unquote(x) for x in args] + fna))
return web.notfound()
def handle(mapping, fvars=None):
"""
Call the appropriate function based on the url to function mapping in `mapping`.
If no module for the function is specified, look up the function in `fvars`. If
`fvars` is empty, using the caller's context.
`mapping` should be a tuple of paired regular expressions with function name
substitutions. `handle` will import modules as necessary.
"""
for url, ofno in utils.group(mapping, 2):
if isinstance(ofno, tuple):
ofn, fna = ofno[0], list(ofno[1:])
else:
ofn, fna = ofno, []
fn, result = utils.re_subm('^' + url + '$', ofn, web.ctx.path)
if result: # it's a match
if fn.split(' ', 1)[0] == "redirect":
url = fn.split(' ', 1)[1]
if web.ctx.method == "GET":
x = web.ctx.env.get('QUERY_STRING', '')
if x:
url += '?' + x
return http.redirect(url)
elif '.' in fn:
x = fn.split('.')
mod, cls = '.'.join(x[:-1]), x[-1]
mod = __import__(mod, globals(), locals(), [""])
cls = getattr(mod, cls)
else:
cls = fn
mod = fvars
if isinstance(mod, types.ModuleType):
mod = vars(mod)
try:
cls = mod[cls]
except KeyError:
return web.notfound()
meth = web.ctx.method
if meth == "HEAD":
if not hasattr(cls, meth):
meth = "GET"
if not hasattr(cls, meth):
return nomethod(cls)
tocall = getattr(cls(), meth)
args = list(result.groups())
for d in re.findall(r'\\(\d+)', ofn):
args.pop(int(d) - 1)
return tocall(*([x and urllib.unquote(x) for x in args] + fna))
return web.notfound()
def __call__(self, hits):
words = getattr(self._cl.query, 'sphinx', self._cl.query)
docs = []
for match in hits.matches:
docs.extend(
[utils._unicode(match['@hit'][f]) for f in self._on_fields])
all_excerpts = self._cl.BuildExcerpts(docs, self._index, words,
self._opts)
print all_excerpts
for match, excerpts in zip(
hits.matches, utils.group(all_excerpts, len(self._on_fields))):
for f, excerpt in zip(self._on_fields, excerpts):
match['@hit'][f + self._suffix] = excerpt or match['@hit'][f]
def _match(self, mapping, value):
for pat, what in utils.group(mapping, 2):
if isinstance(what, application):
if value.startswith(pat):
f = lambda: self._delegate_sub_application(pat, what)
return f, None
else:
continue
elif isinstance(what, basestring):
what, result = utils.re_subm('^' + pat + '$', what, value)
else:
result = utils.re_compile('^' + pat + '$').match(value)
if result: # it's a match
return what, [x and urllib.unquote(x) for x in result.groups()]
return None, None
def summarizeTicks(allTicks, extractKey, itemsEnd, maxCount):
asks = np.zeros(maxCount)
for key, g in group(extractKey, allTicks):
logger.debug('New group, size={size}, key={key}, start={start}'
.format(size=len(g), key=key, start=g[0].date))
n = len(g)
for i in range(0, n):
asks[i] = g[i].ask
date = datetime.datetime(*key)
itemsEnd = min(n, itemsEnd)
yield Summary(date=date,
askMax=np.max(asks[:n]),
askMin=np.min(asks[:n]),
askAverage=np.average(asks[:n]),
askOpen=np.mean(asks[0:itemsEnd]),
askClose=np.mean(asks[n-itemsEnd:n]))
def _match(self, mapping, value):
for pat, what in utils.group(mapping, 2):
if isinstance(what, application):
if value.startswith(pat):
f = lambda: self._delegate_sub_application(pat, what)
return f, None
else:
continue
elif isinstance(what, basestring):
what, result = utils.re_subm('^' + pat + '$', what, value)
else:
result = utils.re_compile('^' + pat + '$').match(value)
if result: # it's a match
return what, [x for x in result.groups()]
return None, None
def profile():
try:
body = group(json.loads(request.data))
interpolated_path = []
for line in body:
interpolated_path.append(interpolate(line[0], line[1]))
result = []
for line in interpolated_path:
for point in line:
alt = int(get_altitude(point[1], point[0]))
result.append([*point, alt])
print(result)
return make_response(json.dumps(result), 200)
except:
return Response(status=404)
def __sub__(self, other):
# TODO subtractionw without index variables
if self.index_type is not None:
index = grizzly_impl.get_field(self.expr, 0)
expr1 = grizzly_impl.get_field(self.expr, 1)
else:
expr1 = self.expr
if other.index_type is not None:
index2 = grizzly_impl.get_field(other.expr, 0)
expr2 = grizzly_impl.get_field(other.expr, 1)
else:
expr2 = other.expr
index_expr = LazyOpResult(index, self.index_type, 0)
sub_expr = SeriesWeld(
grizzly_impl.element_wise_op(expr1, expr2, "-", self.weld_type),
self.weld_type, self.df, self.column_name)
index_sub_expr = utils.group([index_expr, sub_expr])
return SeriesWeld(index_sub_expr.expr, self.weld_type, self.df,
self.column_name, self.index_type, self.index_name)
def evaluate(self, verbose=True, passes=None):
"""Summary
Returns:
TYPE: Description
"""
i = 0
exprs = []
for column_name in self.column_names:
if len(self.column_names) > 1:
index = "1.$%d" % i
else:
index = "1"
expr = self.get_column(column_name,
self.column_types[i],
index,
verbose=verbose)
i += 1
exprs.append(expr)
i = 0
for column_name in self.grouping_column_name:
if len(self.grouping_column_name) > 1:
index = "0.$%d" % i
else:
index = "0"
expr = self.get_column(column_name,
self.grouping_column_types[i],
index,
verbose=verbose)
exprs.append(expr)
i += 1
result = utils.group(exprs).evaluate(verbose=verbose, passes=passes)
df = pd.DataFrame(columns=[])
all_columns = self.column_names + self.grouping_column_name
for i, column_name in enumerate(all_columns):
df[column_name] = result[i]
return dataframeweld.DataFrameWeld(df)
def std(self):
"""Standard deviation
Note that is by default normalizd by n - 1
# TODO, what does pandas do for multiple grouping columns?
# Currently we are just going to use one grouping column
"""
std_expr = grizzly_impl.groupby_std([self.column], [self.column_type],
self.grouping_columns,
self.grouping_column_types)
unzipped_columns = grizzly_impl.unzip_columns(
std_expr,
self.grouping_column_types + [WeldDouble()],
)
index_expr = LazyOpResult(grizzly_impl.get_field(unzipped_columns, 0),
self.grouping_column_types[0], 1)
column_expr = LazyOpResult(grizzly_impl.get_field(unzipped_columns, 1),
self.grouping_column_types[0], 1)
group_expr = utils.group([index_expr, column_expr])
return SeriesWeld(group_expr.expr,
WeldDouble(),
index_type=self.grouping_column_types[0],
index_name=self.grouping_column_names[0])
def parse(self, node):
"""
parse a layer element
"""
from utils import group
from itertools import product, imap
from struct import unpack
import array
self.set_properties(node)
data = None
next_gid = None
data_node = node.find('data')
encoding = data_node.get("encoding", None)
if encoding == "base64":
from base64 import decodestring
data = decodestring(data_node.text.strip())
elif encoding == "csv":
next_gid = imap(
int, "".join(line.strip()
for line in data_node.text.strip()).split(","))
elif encoding:
msg = "TMX encoding type: {0} is not supported."
raise Exception, msg.format(encoding)
compression = data_node.get("compression", None)
if compression == "gzip":
from StringIO import StringIO
import gzip
fh = gzip.GzipFile(fileobj=StringIO(data))
data = fh.read()
fh.close()
elif compression == "zlib":
import zlib
data = zlib.decompress(data)
elif compression:
msg = "TMX compression type: {0} is not supported."
raise Exception, msg.format(str(attr["compression"]))
# if data is None, then it was not decoded or decompressed, so
# we assume here that it is going to be a bunch of tile elements
# TODO: this will probably raise an exception if there are no tiles
if encoding == next_gid is None:
def get_children(parent):
for child in parent.findall('tile'):
yield int(child.get('gid'))
next_gid = get_children(data_node)
elif data:
# data is a list of gids. cast as 32-bit ints to format properly
# create iterator to efficiently parse data
next_gid = imap(lambda i: unpack("<L", "".join(i))[0],
group(data, 4))
# using bytes here limits the layer to 256 unique tiles
# may be a limitation for very detailed maps, but most maps are not
# so detailed.
[self.data.append(array.array("H")) for i in xrange(self.height)]
for (y, x) in product(xrange(self.height), xrange(self.width)):
self.data[y].append(
self.parent.register_gid(*decode_gid(next(next_gid))))
def init_mapping(self, mapping):
self.mapping = list(utils.group(mapping, 2))
def init_mapping(self, mapping):
self.mapping = list(utils.group(mapping, 2))
def parse(self, node):
"""
parse a layer element
"""
from utils import group
from itertools import product, imap
from struct import unpack
import array
self.set_properties(node)
data = None
next_gid = None
data_node = node.find('data')
encoding = data_node.get("encoding", None)
if encoding == "base64":
from base64 import decodestring
data = decodestring(data_node.text.strip())
elif encoding == "csv":
next_gid = imap(int, "".join(
line.strip() for line in data_node.text.strip()
).split(","))
elif encoding:
msg = "TMX encoding type: {0} is not supported."
raise Exception, msg.format(encoding)
compression = data_node.get("compression", None)
if compression == "gzip":
from StringIO import StringIO
import gzip
fh = gzip.GzipFile(fileobj=StringIO(data))
data = fh.read()
fh.close()
elif compression == "zlib":
import zlib
data = zlib.decompress(data)
elif compression:
msg = "TMX compression type: {0} is not supported."
raise Exception, msg.format(str(attr["compression"]))
# if data is None, then it was not decoded or decompressed, so
# we assume here that it is going to be a bunch of tile elements
# TODO: this will probably raise an exception if there are no tiles
if encoding == next_gid is None:
def get_children(parent):
for child in parent.findall('tile'):
yield int(child.get('gid'))
next_gid = get_children(data_node)
elif data:
# data is a list of gids. cast as 32-bit ints to format properly
# create iterator to efficiently parse data
next_gid = imap(lambda i: unpack("<L", "".join(i))[0], group(data, 4))
# using bytes here limits the layer to 256 unique tiles
# may be a limitation for very detailed maps, but most maps are not
# so detailed.
[self.data.append(array.array("H")) for i in xrange(self.height)]
for (y, x) in product(xrange(self.height), xrange(self.width)):
self.data[y].append(self.parent.register_gid(*decode_gid(next(next_gid))))
def test():
import sys
verbose = '-v' in sys.argv
def assertEqual(a, b):
if a == b:
if verbose:
sys.stderr.write('.')
sys.stderr.flush()
else:
assert a == b, "\nexpected: %s\ngot: %s" % (repr(b), repr(a))
from utils import storage, group
t = Template
tests = [
lambda: t('1')(),
'1\n',
lambda: t('$def with ()\n1')(),
'1\n',
lambda: t('$def with (a)\n$a')(1),
'1\n',
lambda: t('$def with (a=0)\n$a')(1),
'1\n',
lambda: t('$def with (a=0)\n$a')(a=1),
'1\n',
lambda: t('$if 1: 1')(),
'1\n',
lambda: t('$if 1:\n 1')(),
'1\n',
lambda: t('$if 0: 0\n$elif 1: 1')(),
'1\n',
lambda: t('$if 0: 0\n$elif None: 0\n$else: 1')(),
'1\n',
lambda: t('$if (0 < 1) and (1 < 2): 1')(),
'1\n',
lambda: t('$for x in [1, 2, 3]: $x')(),
'1\n2\n3\n',
lambda: t('$for x in []: 0\n$else: 1')(),
'1\n',
lambda: t('$def with (a)\n$while a and a.pop(): 1')([1, 2, 3]),
'1\n1\n1\n',
lambda: t('$while 0: 0\n$else: 1')(),
'1\n',
lambda: t('$ a = 1\n$a')(),
'1\n',
lambda: t('$# 0')(),
'',
lambda: t('$def with (d)\n$for k, v in d.iteritems(): $k')({
1: 1
}),
'1\n',
lambda: t('$def with (a)\n$(a)')(1),
'1\n',
lambda: t('$def with (a)\n$a')(1),
'1\n',
lambda: t('$def with (a)\n$a.b')(storage(b=1)),
'1\n',
lambda: t('$def with (a)\n$a[0]')([1]),
'1\n',
lambda: t('${0 or 1}')(),
'1\n',
lambda: t('$ a = [1]\n$a[0]')(),
'1\n',
lambda: t('$ a = {1: 1}\n$a.keys()[0]')(),
'1\n',
lambda: t('$ a = []\n$if not a: 1')(),
'1\n',
lambda: t('$ a = {}\n$if not a: 1')(),
'1\n',
lambda: t('$ a = -1\n$a')(),
'-1\n',
lambda: t('$ a = "1"\n$a')(),
'1\n',
lambda: t('$if 1 is 1: 1')(),
'1\n',
lambda: t('$if not 0: 1')(),
'1\n',
lambda: t('$if 1:\n $if 1: 1')(),
'1\n',
lambda: t('$ a = 1\n$a')(),
'1\n',
lambda: t('$ a = 1.\n$a')(),
'1.0\n',
lambda: t('$({1: 1}.keys()[0])')(),
'1\n',
]
for func, value in group(tests, 2):
assertEqual(func(), value)
j = Template("$var foo: bar")()
assertEqual(str(j), '')
assertEqual(j.foo, 'bar\n')
if verbose: sys.stderr.write('\n')
def group_by_mean(result):
# group by training seed and reduce by mean
result = utils.group(result, 5)
result = utils.reduce_by_mean(result)
return result
def group_by_max(result):
result = utils.group(result, 5)
result = utils.reduce_by_max_val(result)
return result
#calculator for loan maturity
#imports
from datetime import datetime
import utils
#####
interests = utils.loaddata(utils.interests)
txs = utils.loaddata(utils.txs)
list_txs = utils.group(txs)
list_interests = utils.group(interests)
del interests, txs
list_txs = utils.reindex(list_txs)
list_interests = utils.reindex(list_interests)
list_txs[7] = list_txs[7].append(list_txs[8])
del list_txs[8]
list_txs.insert(6, list_txs[3])
del list_txs[3]
symbols = []
comp_interest = []
apys = []
for i in list_interests:
symbols.append(i['SYMBOL'][0])
comp_interest.append(i['COMP'].mean())
apys.append(i['APY'].mean())
####
def add_mapping(self, map):
self.mapping.extend(utils.group(map, 2))
def get_bureau(path, num_rows=None):
"""Preprocess and extract features from bureau and bureau balance.
Get bureau balance features grouped by SK_ID_BUREAU and append to
bureau data. After that, it performs aggregations for each customer
(unique SK_ID_CURR) and return a DataFrame
Arguments:
path: Path to the folder where files are saved (string).
num_rows: Number of rows to read; None reads all rows (int, default: None).
Returns:
df: DataFrame with processed data.
"""
bureau = pd.read_csv(os.path.join(path, 'bureau.csv'), nrows= num_rows)
# Credit duration and credit/account end date difference
bureau['CREDIT_DURATION'] = -bureau['DAYS_CREDIT'] + bureau['DAYS_CREDIT_ENDDATE']
bureau['ENDDATE_DIF'] = bureau['DAYS_CREDIT_ENDDATE'] - bureau['DAYS_ENDDATE_FACT']
# Credit to debt ratio and difference
bureau['DEBT_PERCENTAGE'] = bureau['AMT_CREDIT_SUM'] / bureau['AMT_CREDIT_SUM_DEBT']
bureau['DEBT_CREDIT_DIFF'] = bureau['AMT_CREDIT_SUM'] - bureau['AMT_CREDIT_SUM_DEBT']
bureau['CREDIT_TO_ANNUITY_RATIO'] = bureau['AMT_CREDIT_SUM'] / bureau['AMT_ANNUITY']
# One-hot encoder
bureau, _ = utils.one_hot_encoder(bureau, nan_as_category= False)
# Join bureau balance features
bureau = bureau.merge(get_bureau_balance(path, num_rows), how='left', on='SK_ID_BUREAU')
# Flag months with late payments (days past due)
bureau['STATUS_12345'] = 0
for i in range(1,6):
bureau['STATUS_12345'] += bureau['STATUS_{}'.format(i)]
# Aggregate by number of months in balance and merge with bureau (loan length agg)
features = ['AMT_CREDIT_MAX_OVERDUE', 'AMT_CREDIT_SUM_OVERDUE', 'AMT_CREDIT_SUM',
'AMT_CREDIT_SUM_DEBT', 'DEBT_PERCENTAGE', 'DEBT_CREDIT_DIFF', 'STATUS_0', 'STATUS_12345']
agg_length = bureau.groupby('MONTHS_BALANCE_SIZE')[features].mean().reset_index()
agg_length.rename({feat: 'LL_' + feat for feat in features}, axis=1, inplace=True)
bureau = bureau.merge(agg_length, how='left', on='MONTHS_BALANCE_SIZE')
del agg_length
gc.collect()
# General loans aggregations
agg_bureau = utils.group(bureau, 'BUREAU_', config.BUREAU_AGG)
# Active and closed loans aggregations
active = bureau[bureau['CREDIT_ACTIVE_Active'] == 1]
agg_bureau = utils.group_and_merge(active,agg_bureau,'BUREAU_ACTIVE_',config.BUREAU_ACTIVE_AGG)
closed = bureau[bureau['CREDIT_ACTIVE_Closed'] == 1]
agg_bureau = utils.group_and_merge(closed,agg_bureau,'BUREAU_CLOSED_',config.BUREAU_CLOSED_AGG)
del active, closed
gc.collect()
# Aggregations for the main loan types
for credit_type in ['Consumer credit', 'Credit card', 'Mortgage', 'Car loan', 'Microloan']:
type_df = bureau[bureau['CREDIT_TYPE_' + credit_type] == 1]
prefix = 'BUREAU_' + credit_type.split(' ')[0].upper() + '_'
agg_bureau = utils.group_and_merge(type_df, agg_bureau, prefix, config.BUREAU_LOAN_TYPE_AGG)
del type_df
gc.collect()
# Time based aggregations: last x months
for time_frame in [6, 12, 24, 36]:
prefix = "BUREAU_LAST{}M_".format(time_frame)
time_frame_df = bureau[bureau['DAYS_CREDIT'] >= -30*time_frame]
agg_bureau = utils.group_and_merge(time_frame_df,agg_bureau,prefix,config.BUREAU_TIME_AGG)
del time_frame_df
gc.collect()
# Last loan max overdue
sort_bureau = bureau.sort_values(by=['DAYS_CREDIT'])
gr = sort_bureau.groupby('SK_ID_CURR')['AMT_CREDIT_MAX_OVERDUE'].last().reset_index()
gr.rename({'AMT_CREDIT_MAX_OVERDUE': 'BUREAU_LAST_LOAN_MAX_OVERDUE'}, inplace=True)
agg_bureau = agg_bureau.merge(gr, on='SK_ID_CURR', how='left')
# Ratios: total debt/total credit and active loans debt/ active loans credit
agg_bureau['BUREAU_DEBT_OVER_CREDIT'] = \
agg_bureau['BUREAU_AMT_CREDIT_SUM_DEBT_SUM']/agg_bureau['BUREAU_AMT_CREDIT_SUM_SUM']
agg_bureau['BUREAU_ACTIVE_DEBT_OVER_CREDIT'] = \
agg_bureau['BUREAU_ACTIVE_AMT_CREDIT_SUM_DEBT_SUM']/agg_bureau['BUREAU_ACTIVE_AMT_CREDIT_SUM_SUM']
return agg_bureau
def mtime_beat():
'''每次任务只跑一年的'''
y_list = []
y = get_year() + 1 # 要抓取的年份
debug('Fetch Year: {} starting...'.format(y))
instance = fetch(y, 1)
page = get_movie_pages(instance)
if page is None:
warn('Movie"page has not fetched')
# 执行间隔自适应
if scheduler.get_interval < TASK_BEAT * 7:
scheduler.change_interval(incr=True)
return
ids = get_movie_ids(instance)
if ids is None:
# 间隔自适应也不能太大
warn('Movie has not fetched')
if scheduler.get_interval < TASK_BEAT * 7:
scheduler.change_interval(incr=True)
return
# 当任务继续能执行的时候,回到默认的间隔
if scheduler.get_interval > TASK_BEAT:
debug('Interval back to default')
scheduler.change_interval(TASK_BEAT)
y_list.extend(ids)
if not y_list:
# 本年没有电影
debug('Year: {} has not movie'.format(y))
YearFinished(year=y).save()
sleep2()
return mtime_beat()
if page > 1:
p = 2
while p <= page:
instance = fetch(y, p)
debug('Fetch Year:{} Page:{}'.format(y, p))
ids = get_movie_ids(instance)
if ids is None:
# 间隔自适应也不能太大
if scheduler.get_interval < TASK_BEAT * 7:
scheduler.change_interval(incr=True)
# 出现需要验证码 手动输入或者等待一段时间后重试,直到能正常使用
sleep2(VERIFY_INTERVAL)
continue
ids = []
y_list.extend(ids)
p += 1
sleep2()
obj = IdFinished.objects(year=y).first()
if obj is not None:
has_finished = obj.ids
else:
has_finished = []
to_process = get_unfinished(has_finished, y_list)
# 给相应队列添加任务
for payload in group(to_process, TASK_BEAT_NUM):
for task in ['Fullcredits', 'Movie', 'Comment', 'Character',
'MicroComment', 'Scenes', 'Awards', 'Plot',
'Details']:
debug('Push payload: {} to {} Queue'.format(payload, task))
try:
Message(year=y, task=task, payload=payload).save()
# Hack一下
#Message.objects.get_or_create(year=y, task=task, payload=payload)
except NotUniqueError:
debug('Duplicate insert: [{}], payload: {}'.format(task, payload))
# 当前年份数据已经入MQ
YearFinished(year=y).save()
debug('Year: {} done'.format(y))
end = args["end_val"]
delta = args["delta"] * 1.0
if start == end:
return 1
if delta > 0 and start > end:
return 0
elif delta < 0 and end > start:
return 0
elif start <= end:
return iceil((end + 1 - start) / abs(delta))
else:
return iceil((start + 1 - end) / abs(delta))
TESTS_GROUPS = group(
flatten([cls().build_tests() for cls in Gbs3TestScript.__subclasses__()]),
128
)
TEST_DIR = os.path.dirname(os.path.dirname(__file__))
THIS_TEST_DIR = os.path.dirname(__file__)
class AutoTestCaseGbs3(unittest.TestCase, AutoTestCase):
def setUp(self):
copy_file(
THIS_TEST_DIR + "/Biblioteca.gbs",
TEST_DIR + "/examples/Biblioteca.gbs"
)
def gobstones_parameters(self):
return "--language gobstones"
def cmd_setup(args):
"""Setup environment."""
if not args.dirs:
args.kernel = False
args.rdma_core = False
args.iproute = False
if args.installs:
print(""" This setup script will update your hypervisor to latest
distribution packages and install docker. Please restart
the hypervisor to complete the installation. """)
if args.yes is False and utils.query_yes_no("Do you want to proceed?",
'no') is False:
exit("Exiting ...")
supported_os = {
'fedora': '26',
'ubuntu': '16',
'rhel': '8',
'redhat': '8',
}
# Python API stability is dissaster
# module platform was deprecated https://docs.python.org/3/library/platform.html
# Luckily enough, they added distro module before removing platform
try:
import distro
distro_id = distro.id()
distro_v = distro.major_version()
except ModuleNotFoundError:
import platform
distro_id = platform.dist()[0].lower()
distro_v = platform.dist()[1].split('.')[0]
if distro_id not in supported_os.keys(
) or distro_v < supported_os[distro_id]:
exit(""" Your hypervisor is not supported. Exiting ...""")
if args.installs:
setuphv = utils.get_internal_fn('scripts/')
if distro_id == 'redhat':
distro_id = 'rhel'
setuphv += 'setup-hv.' + distro_id
subprocess.check_call(setuphv)
utils.init_config_file()
section = utils.load_config_file()
if args.dirs:
for key, value in section.items():
if args.force:
subprocess.call(["sudo", "rm", "-rf", value])
if os.path.exists(value):
exit("Please remove " + value + " Exiting ...")
if key == "kernel" and not args.kernel:
continue
if key == "rdma-core" and not args.rdma_core:
continue
if key == "iproute2" and not args.iproute:
continue
if key == 'os':
continue
print("Prepare " + key)
subprocess.call(["sudo", "mkdir", "-p", value])
subprocess.call([
"sudo", "chown", "-R",
utils.username() + ":" + utils.group(), value
])
if key == "src" or key == "logs" or key == "ccache":
continue
if key == "kernel":
key = "linux"
p = subprocess.Popen([
"git", "clone", "ssh://" + utils.username() +
"@l-gerrit.mtl.labs.mlnx:29418/upstream/" + key, "."
],
cwd=value)
p.wait()
p = subprocess.Popen([
"scp", "-p", "-P", "29418",
utils.username() + "@l-gerrit.mtl.labs.mlnx:hooks/commit-msg",
".git/hooks/"
],
cwd=value)
p.wait()
if key == "linux":
shutil.copy(
os.path.join(os.path.dirname(__file__),
"../configs/kconfig-kvm"), value + "/.config")
print("Completed, PLEASE RESTART server")
