def predict(self, dataset, predict_set_name=None):
"""
Parameters:
dataset: string
Dataset name to use for testing
predict_set_name: string (default None)
Name to give to the dataset containing the predictions. If None,
a randomly generated name will be given
"""
if predict_set_name is None:
predict_set_name = generate_random_name()
self.predict_payload = Transform(
inputData="""
SELECT
%(func)s({%(feature)s as score}) as predict
FROM %(test)s
""" % {
"func": self.name,
"feature": self.feature,
"test": dataset
},
outputDataset=predict_set_name
)()
response = mldb.connection.put(
"/v1/procedures/train_test_split",
self.predict_payload
)
if response.status_code != 201:
raise Exception("could not create dataset.\n{}".format(
response.content))
return predict_set_name
def __init__(self, use_website = False, cache_dir = None, version = 'CURRENT'):
self.base_url = "http://rfam.xfam.org/"
self.use_website = use_website
if not cache_dir:
self.cache_dir = "/tmp/RFAM_%s"%utils.generate_random_name(7)
else:
self.cache_dir = cache_dir
self.version = version
if not os.path.exists(self.cache_dir):
shutil.os.mkdir(self.cache_dir)
def attach_volume(self, service_name, vm_name, size, lun):
disk_name = utils.generate_random_name(5, vm_name)
media_link = self._get_media_link(vm_name, disk_name)
self.sms.add_data_disk(service_name,
vm_name,
vm_name,
lun,
host_caching='ReadWrite',
media_link=media_link,
disk_name=disk_name,
logical_disk_size_in_gb=size)
def stratified_sample(dataset, col, weights, outputDataset=None):
"""
Stratification is the process of dividing members of the population into
homogeneous subgroups before sampling. The strata should be mutually
exclusive: every element in the population must be assigned to only one
stratum.
Sampling each stratum can be done proportionnally or not. You could want
to keep the same proportions between your strata or unbalance it.
https://en.wikipedia.org/wiki/Stratified_sampling
Paramters:
dataset: string
Name of the dataset to sample.
col: string (default None)
Name of the column containing the class
weights: dict
A dictionnary containing the class as key and the number of rows
to sample as value.
e.g. If you have class A and B and you want respectively 1000 and
500, this parameter would be {'A': 1000, 'B':500}
outputDataset: OutputDataset (default None)
"""
if outputDataset is None:
outputDataset = _create_output_dataset(generate_random_name())
q = """
SELECT *
FROM sample(%(dataset)s, {rows: %(value)s, withReplacement: FALSE})
WHERE %(col)s = '%(key)s'
""" % {"dataset": dataset, "col": col}
subselects = []
for key, value in weights.items():
subselects.append(q % {"key": key, "value": value})
merge = "SELECT * FROM merge({})".format(",".join(subselects))
mldb.connection.put(
"/v1/procedures/stratifiedSample",
Transform(
inputData=merge,
outputData=outputDataset
)()
)
def _get_or_create_storage_account(self):
account_list = self.sms.list_storage_accounts()
if account_list:
return account_list[-1].service_name
storage_account = utils.generate_random_name(10)
description = "Storage account %s description" % storage_account
label = storage_account + 'label'
self.sms.create_storage_account(storage_account,
description,
label,
location=self.location)
return storage_account
def update(request):
"""
Update user data
"""
session = utils.check_auth_token(request)
if not session:
return utils.invalid_auth_response()
user = session.user
display_name = request.DATA.get('display_name', '').strip()
photo_file = request.FILES.get('photo', None)
if display_name:
user.first_name = display_name.lower().title()
if photo_file:
origin_name, ext = os.path.splitext(photo_file.name)
if not ext:
ext = "jpg"
aws_name = utils.slugify_name(user.username) + '/' + utils.generate_random_name(origin_name)
aws_name += ext
photo = utils.add_object_via_cloudfront(aws_name, photo_file, False)
if photo:
try:
old_photo = user.profile.photo_s3_name
utils.delete_s3_object(old_photo, False)
except AttributeError:
pass
user.profile.display_photo = photo
user.profile.photo_s3_name = aws_name
user.profile.save()
else:
msg = "Can't upload display photo to S3"
msg_code = "failed"
return utils.failed_response(msg, msg_code)
user.save()
msg = "User %s was updated" %user.username
msg_code = "ok"
return utils.successful_response(msg,
msg_code,
display_name=user.first_name,
email=user.email,
photo_url=user.profile.display_photo)
def test_get_content_with_param_from_address_string(self):
# Create new page for test
name = "Page_" + utils.generate_random_name()
data = {}
data["Name"] = name
data["Value"] = "#test#"
data["Conditions"] = "true"
data["Menu"] = "default_menu"
data["ApplicationId"] = 1
res = self.call("NewPage", data)
self.assertGreater(int(res), 0, "BlockId is not generated: " + res)
# Test
param = "?test="
value = "hello123"
asserts = ["tree"]
res = self.check_post_api("/content/page/" + name + param + value, "",
asserts)
self.assertEqual(value, res["tree"][0]["text"])
def test_z1_dbSelectMetricsMin(self):
# func generate contract which return block_id and increment count blocks
def waitBlockId(old_block_id, limit):
while True:
if old_block_id == limit:
break
contracName = utils.generate_random_name()
value = "contract con_" + contracName + " {\n data{} \n conditions{} \n action { \n $result = $block \n } \n }"
data = {
"Value": value,
"ApplicationId": 1,
"Conditions": "true"
}
result = utils.call_contract(url, prKey, "NewContract", data,
token)
tx = utils.txstatus(url,
self.config["1"]["time_wait_tx_in_block"],
result['hash'], token)
current_block_id = int(tx["blockid"])
self.assertGreater(current_block_id, 0,
"BlockId is not generated: " + str(tx))
old_block_id = current_block_id
# generate contract which return count blocks in blockchain
contracName = utils.generate_random_name()
value = "contract con_" + contracName + " {\n data{} \n conditions{} \n action { \n $result = $block \n } \n }"
data = {"Value": value, "ApplicationId": 1, "Conditions": "true"}
result = utils.call_contract(url, prKey, "NewContract", data, token)
tx = utils.txstatus(url, self.config["1"]["time_wait_tx_in_block"],
result['hash'], token)
current_block_id = int(tx["blockid"])
self.assertGreater(current_block_id, 0,
"BlockId is not generated: " + str(tx))
# wait until generated 100 blocks
if current_block_id < 100:
waitBlockId(current_block_id, 100)
# wait until generated multiples of 100 blocks
if (current_block_id % 100 >= 90):
count = current_block_id + (100 - (current_block_id % 100))
waitBlockId(current_block_id, count)
# test
ecosystem_tx = self.getMetrics(1, "ecosystem_tx")
contract = self.contracts["dbSelectMetricsMin"]
self.check_contract(contract["code"], str(ecosystem_tx))
def addUserTable(self):
# add table
column = """[{"name":"MyName","type":"varchar",
"index": "1", "conditions":"true"},
{"name":"ver_on_null","type":"varchar",
"index": "1", "conditions":"true"}]"""
permission = """{"read": "true",
"insert": "true",
"update": "true",
"new_column": "true"}"""
tableName = "rolltab_" + utils.generate_random_name()
data = {
"Name": tableName,
"Columns": column,
"ApplicationId": 1,
"Permissions": permission
}
res = self.call("NewTable", data)
return tableName
def predict(self, dataset, predict_set_name=None):
"""
Predict class for X.
The predicted class of an input sample is a vote by the trees in the
forest, weighted by their probability estimates. That is, the predicted
class is the one with highest mean probability estimate across the
trees.
Parameters:
dataset: string
Dataset name to use for testing
predict_set_name: string (default None)
Name to give to the dataset containing the predictions. If None,
a randomly generated name will be given
"""
if predict_set_name is None:
predict_set_name = generate_random_name()
self.predict_payload = Transform(
inputData="""
SELECT
%(func)s({{%(features)s} as features}) as predict
FROM %(test)s
""" % {
"func": self.name,
"features": ",".join(self.features),
"test": dataset
},
outputDataset=predict_set_name
)()
response = mldb.connection.put(
"/v1/procedures/train_test_split",
self.predict_payload
)
if response.status_code != 201:
raise Exception("could not create dataset.\n{}".format(
response.content))
return predict_set_name
def waitBlockId(old_block_id, limit):
while True:
if old_block_id == limit:
break
contracName = utils.generate_random_name()
value = "contract con_" + contracName + " {\n data{} \n conditions{} \n action { \n $result = $block \n } \n }"
data = {
"Value": value,
"ApplicationId": 1,
"Conditions": "true"
}
result = utils.call_contract(url, prKey, "NewContract", data,
token)
tx = utils.txstatus(url,
self.config["1"]["time_wait_tx_in_block"],
result['hash'], token)
current_block_id = int(tx["blockid"])
self.assertGreater(current_block_id, 0,
"BlockId is not generated: " + str(tx))
old_block_id = current_block_id
def test_max_block_size(self):
max_block_size = utils.get_system_parameter(conf["1"]["dbHost"],
conf["1"]["dbName"],
conf["1"]["login"],
conf["1"]["pass"],
"max_block_size")
self.update_sys_param("max_block_size", "500")
name = "cont" + utils.generate_random_name()
code = "contract " + name + contract["limits"]["code"]
data = {
"Wallet": "",
"Value": code,
"ApplicationId": 1,
"Conditions": "true"
}
error = self.call("NewContract", data)
self.assertEqual(error, "stop generating block",
"Incorrect error: " + error)
self.update_sys_param("max_block_size", str(max_block_size))
time.sleep(30)
def test_getHistoryPage(self):
# create page
name = utils.generate_random_name()
page = "Div(Body: Hello)"
data = {
"ApplicationId": "1",
"Name": name,
"Value": page,
"Menu": "default_menu",
"Conditions": "true"
}
self.call_contract("NewPage", data)
# change page
id = funcs.get_object_id(url, name, "pages", token)
newValuePage = page.replace("Hello", "new_var")
data = {"Id": id, "Value": newValuePage}
self.call_contract("EditPage", data)
# test
data = {"Table": "pages", "ID": id}
contract = self.contracts["getHistory"]
self.check_contract_with_data(contract["code"], data, page)
def test_getHistoryRowMenu(self):
# create menu
rollc_before = utils.getMaxIdFromTable(dbHost, dbName, login, pas,
"rollback_tx")
name = utils.generate_random_name()
menu = "This is new menu"
data = {"Name": name, "Value": menu, "Conditions": "true"}
self.call_contract("NewMenu", data)
rollc_after = utils.getMaxIdFromTable(dbHost, dbName, login, pas,
"rollback_tx")
# change menu
id = funcs.get_object_id(url, name, "menu", token)
newValueMenu = menu.replace("new menu", "new_var")
data = {"Id": id, "Value": newValueMenu}
self.call_contract("EditMenu", data)
# test
query = """SELECT id FROM "rollback_tx" WHERE table_name = '1_menu' AND data='' AND id >= %s AND id <= %s""" % (
rollc_before, rollc_after)
rollback_id = utils.executeSQL(dbHost, dbName, login, pas, query)[0][0]
data = {"Table": "menu", "ID": id, "rID": rollback_id}
contract = self.contracts["getHistoryRow"]
self.check_contract_with_data(contract["code"], data, menu)
def form_valid(self, form):
name = form.cleaned_data['name']
file = form.cleaned_data['file']
try:
origin_name, ext = os.path.splitext(file.name)
aws_name = 'pinna_ads/'\
+ utils.generate_random_name(origin_name) + ext
url = utils.add_object_via_cloudfront(aws_name, file, False)
models.PinnaAd.objects.create(name=name, url=url, s3_name=aws_name)
messages.add_message(self.request,
messages.INFO,
"Created new AD successfully")
except Exception as e:
LOG.error(traceback.format_exc())
messages.add_message(self.request,
messages.ERROR,
"Can't create AD: %s" %e)
return super(AdCreate, self).form_valid(form)
def test_get_avatar_without_login(self):
# add file in binaries
name = "file_" + utils.generate_random_name()
path = os.path.join(os.getcwd(), "fixtures", "image2.jpg")
with open(path, 'rb') as f:
file = f.read()
files = {'Data': file}
data = {"Name": name, "ApplicationId": 1, "DataMimeType": "image/jpeg"}
resp = utils.call_contract_with_files(url, prKey, "UploadBinary", data,
files, token)
res = self.assertTxInBlock(resp, token)
self.assertGreater(int(res), 0, "BlockId is not generated: " + res)
# find last added file
asserts = ["count"]
res = self.check_get_api("/list/binaries", "", asserts)
lastRec = res["count"]
# find founder ID
asserts = ["list"]
res = self.check_get_api("/list/members", "", asserts)
# iterating response elements
i = 0
founderID = ""
while i < len(res['list']):
if res['list'][i]['member_name'] == "founder":
founderID = res['list'][i]['id']
i += 1
# change column permissions
data = {
"TableName": "members",
"Name": "image_id",
"UpdatePerm": "true",
"ReadPerm": "true"
}
res = self.call("EditColumn", data)
self.assertGreater(int(res), 0, "BlockId is not generated: " + res)
# update members table
code = """
{
data{}
conditions{}
action{
DBUpdate("members", %s, {image_id: "%s"})
}
}
""" % (founderID, lastRec)
code, name = utils.generate_name_and_code(code)
data = {"Value": code, "ApplicationId": 1, "Conditions": "true"}
res = self.call("NewContract", data)
self.assertGreater(int(res), 0, "BlockId is not generated: " + res)
data = {}
resp = utils.call_contract(url, prKey, name, data, token)
res = self.assertTxInBlock(resp, token)
self.assertGreater(int(res), 0, "BlockId is not generated: " + res)
# rollback changes column permissions
data = {
"TableName": "members",
"Name": "image_id",
"UpdatePerm": "ContractAccess(\"Profile_Edit\")",
"ReadPerm": "true"
}
res = self.call("EditColumn", data)
self.assertGreater(int(res), 0, "BlockId is not generated: " + res)
# test
ecosystemID = "1"
avaURL = url + "/avatar/" + ecosystemID + "/" + founderID
resp = requests.get(avaURL)
msg = "Content-Length is different!"
self.assertIn("71926", str(resp.headers["Content-Length"]), msg)
def new_parameter(self):
name = "Par_" + utils.generate_random_name()
data = {"Name": name, "Value": "test", "Conditions": "true"}
res = self.call("NewParameter", data)
return name
def create_ecosystem(self):
data = {"Name": "Ecosys" + utils.generate_random_name()}
res = self.call("NewEcosystem", data)
def generate_cloud_service_name(self, os_user=None, random=False):
if random:
return utils.generate_random_name(10)
return '-'.join((os_user, utils.generate_random_name(6)))
def generate_cloud_service_name(self, os_user=None, random=False):
if random:
return utils.generate_random_name(10)
return '-'.join((os_user, utils.generate_random_name(6)))
def test_get_ecosystem_name_new(self):
data = {"Name": "ecos_" + utils.generate_random_name()}
res = self.call("NewEcosystem", data)
id = self.check_get_api("/list/ecosystems", "", [])["count"]
asserts = ["ecosystem_name"]
self.check_get_api("/ecosystemname?id=" + str(id), "", asserts)
def new_menu(self, url, prKey):
name = "Menu_" + utils.generate_random_name()
data = {"Name": name, "Value": "Item1", "Conditions": "true"}
res = utils.call_contract(url, prKey, "NewMenu", data,
self.data1["jwtToken"])
return funcs.get_count(url, "menu", self.data1["jwtToken"])
def generate_name_and_code(self, sourseCode):
name = utils.generate_random_name()
code = "contract " + name + sourseCode
return code, name