def predict(self, dataset, predict_set_name=None): """ Parameters: dataset: string Dataset name to use for testing predict_set_name: string (default None) Name to give to the dataset containing the predictions. If None, a randomly generated name will be given """ if predict_set_name is None: predict_set_name = generate_random_name() self.predict_payload = Transform( inputData=""" SELECT %(func)s({%(feature)s as score}) as predict FROM %(test)s """ % { "func": self.name, "feature": self.feature, "test": dataset }, outputDataset=predict_set_name )() response = mldb.connection.put( "/v1/procedures/train_test_split", self.predict_payload ) if response.status_code != 201: raise Exception("could not create dataset.\n{}".format( response.content)) return predict_set_name
def __init__(self, use_website = False, cache_dir = None, version = 'CURRENT'): self.base_url = "http://rfam.xfam.org/" self.use_website = use_website if not cache_dir: self.cache_dir = "/tmp/RFAM_%s"%utils.generate_random_name(7) else: self.cache_dir = cache_dir self.version = version if not os.path.exists(self.cache_dir): shutil.os.mkdir(self.cache_dir)
def attach_volume(self, service_name, vm_name, size, lun): disk_name = utils.generate_random_name(5, vm_name) media_link = self._get_media_link(vm_name, disk_name) self.sms.add_data_disk(service_name, vm_name, vm_name, lun, host_caching='ReadWrite', media_link=media_link, disk_name=disk_name, logical_disk_size_in_gb=size)
def stratified_sample(dataset, col, weights, outputDataset=None): """ Stratification is the process of dividing members of the population into homogeneous subgroups before sampling. The strata should be mutually exclusive: every element in the population must be assigned to only one stratum. Sampling each stratum can be done proportionnally or not. You could want to keep the same proportions between your strata or unbalance it. https://en.wikipedia.org/wiki/Stratified_sampling Paramters: dataset: string Name of the dataset to sample. col: string (default None) Name of the column containing the class weights: dict A dictionnary containing the class as key and the number of rows to sample as value. e.g. If you have class A and B and you want respectively 1000 and 500, this parameter would be {'A': 1000, 'B':500} outputDataset: OutputDataset (default None) """ if outputDataset is None: outputDataset = _create_output_dataset(generate_random_name()) q = """ SELECT * FROM sample(%(dataset)s, {rows: %(value)s, withReplacement: FALSE}) WHERE %(col)s = '%(key)s' """ % {"dataset": dataset, "col": col} subselects = [] for key, value in weights.items(): subselects.append(q % {"key": key, "value": value}) merge = "SELECT * FROM merge({})".format(",".join(subselects)) mldb.connection.put( "/v1/procedures/stratifiedSample", Transform( inputData=merge, outputData=outputDataset )() )
def _get_or_create_storage_account(self): account_list = self.sms.list_storage_accounts() if account_list: return account_list[-1].service_name storage_account = utils.generate_random_name(10) description = "Storage account %s description" % storage_account label = storage_account + 'label' self.sms.create_storage_account(storage_account, description, label, location=self.location) return storage_account
def update(request): """ Update user data """ session = utils.check_auth_token(request) if not session: return utils.invalid_auth_response() user = session.user display_name = request.DATA.get('display_name', '').strip() photo_file = request.FILES.get('photo', None) if display_name: user.first_name = display_name.lower().title() if photo_file: origin_name, ext = os.path.splitext(photo_file.name) if not ext: ext = "jpg" aws_name = utils.slugify_name(user.username) + '/' + utils.generate_random_name(origin_name) aws_name += ext photo = utils.add_object_via_cloudfront(aws_name, photo_file, False) if photo: try: old_photo = user.profile.photo_s3_name utils.delete_s3_object(old_photo, False) except AttributeError: pass user.profile.display_photo = photo user.profile.photo_s3_name = aws_name user.profile.save() else: msg = "Can't upload display photo to S3" msg_code = "failed" return utils.failed_response(msg, msg_code) user.save() msg = "User %s was updated" %user.username msg_code = "ok" return utils.successful_response(msg, msg_code, display_name=user.first_name, email=user.email, photo_url=user.profile.display_photo)
def test_get_content_with_param_from_address_string(self): # Create new page for test name = "Page_" + utils.generate_random_name() data = {} data["Name"] = name data["Value"] = "#test#" data["Conditions"] = "true" data["Menu"] = "default_menu" data["ApplicationId"] = 1 res = self.call("NewPage", data) self.assertGreater(int(res), 0, "BlockId is not generated: " + res) # Test param = "?test=" value = "hello123" asserts = ["tree"] res = self.check_post_api("/content/page/" + name + param + value, "", asserts) self.assertEqual(value, res["tree"][0]["text"])
def test_z1_dbSelectMetricsMin(self): # func generate contract which return block_id and increment count blocks def waitBlockId(old_block_id, limit): while True: if old_block_id == limit: break contracName = utils.generate_random_name() value = "contract con_" + contracName + " {\n data{} \n conditions{} \n action { \n $result = $block \n } \n }" data = { "Value": value, "ApplicationId": 1, "Conditions": "true" } result = utils.call_contract(url, prKey, "NewContract", data, token) tx = utils.txstatus(url, self.config["1"]["time_wait_tx_in_block"], result['hash'], token) current_block_id = int(tx["blockid"]) self.assertGreater(current_block_id, 0, "BlockId is not generated: " + str(tx)) old_block_id = current_block_id # generate contract which return count blocks in blockchain contracName = utils.generate_random_name() value = "contract con_" + contracName + " {\n data{} \n conditions{} \n action { \n $result = $block \n } \n }" data = {"Value": value, "ApplicationId": 1, "Conditions": "true"} result = utils.call_contract(url, prKey, "NewContract", data, token) tx = utils.txstatus(url, self.config["1"]["time_wait_tx_in_block"], result['hash'], token) current_block_id = int(tx["blockid"]) self.assertGreater(current_block_id, 0, "BlockId is not generated: " + str(tx)) # wait until generated 100 blocks if current_block_id < 100: waitBlockId(current_block_id, 100) # wait until generated multiples of 100 blocks if (current_block_id % 100 >= 90): count = current_block_id + (100 - (current_block_id % 100)) waitBlockId(current_block_id, count) # test ecosystem_tx = self.getMetrics(1, "ecosystem_tx") contract = self.contracts["dbSelectMetricsMin"] self.check_contract(contract["code"], str(ecosystem_tx))
def addUserTable(self): # add table column = """[{"name":"MyName","type":"varchar", "index": "1", "conditions":"true"}, {"name":"ver_on_null","type":"varchar", "index": "1", "conditions":"true"}]""" permission = """{"read": "true", "insert": "true", "update": "true", "new_column": "true"}""" tableName = "rolltab_" + utils.generate_random_name() data = { "Name": tableName, "Columns": column, "ApplicationId": 1, "Permissions": permission } res = self.call("NewTable", data) return tableName
def predict(self, dataset, predict_set_name=None): """ Predict class for X. The predicted class of an input sample is a vote by the trees in the forest, weighted by their probability estimates. That is, the predicted class is the one with highest mean probability estimate across the trees. Parameters: dataset: string Dataset name to use for testing predict_set_name: string (default None) Name to give to the dataset containing the predictions. If None, a randomly generated name will be given """ if predict_set_name is None: predict_set_name = generate_random_name() self.predict_payload = Transform( inputData=""" SELECT %(func)s({{%(features)s} as features}) as predict FROM %(test)s """ % { "func": self.name, "features": ",".join(self.features), "test": dataset }, outputDataset=predict_set_name )() response = mldb.connection.put( "/v1/procedures/train_test_split", self.predict_payload ) if response.status_code != 201: raise Exception("could not create dataset.\n{}".format( response.content)) return predict_set_name
def waitBlockId(old_block_id, limit): while True: if old_block_id == limit: break contracName = utils.generate_random_name() value = "contract con_" + contracName + " {\n data{} \n conditions{} \n action { \n $result = $block \n } \n }" data = { "Value": value, "ApplicationId": 1, "Conditions": "true" } result = utils.call_contract(url, prKey, "NewContract", data, token) tx = utils.txstatus(url, self.config["1"]["time_wait_tx_in_block"], result['hash'], token) current_block_id = int(tx["blockid"]) self.assertGreater(current_block_id, 0, "BlockId is not generated: " + str(tx)) old_block_id = current_block_id
def test_max_block_size(self): max_block_size = utils.get_system_parameter(conf["1"]["dbHost"], conf["1"]["dbName"], conf["1"]["login"], conf["1"]["pass"], "max_block_size") self.update_sys_param("max_block_size", "500") name = "cont" + utils.generate_random_name() code = "contract " + name + contract["limits"]["code"] data = { "Wallet": "", "Value": code, "ApplicationId": 1, "Conditions": "true" } error = self.call("NewContract", data) self.assertEqual(error, "stop generating block", "Incorrect error: " + error) self.update_sys_param("max_block_size", str(max_block_size)) time.sleep(30)
def test_getHistoryPage(self): # create page name = utils.generate_random_name() page = "Div(Body: Hello)" data = { "ApplicationId": "1", "Name": name, "Value": page, "Menu": "default_menu", "Conditions": "true" } self.call_contract("NewPage", data) # change page id = funcs.get_object_id(url, name, "pages", token) newValuePage = page.replace("Hello", "new_var") data = {"Id": id, "Value": newValuePage} self.call_contract("EditPage", data) # test data = {"Table": "pages", "ID": id} contract = self.contracts["getHistory"] self.check_contract_with_data(contract["code"], data, page)
def test_getHistoryRowMenu(self): # create menu rollc_before = utils.getMaxIdFromTable(dbHost, dbName, login, pas, "rollback_tx") name = utils.generate_random_name() menu = "This is new menu" data = {"Name": name, "Value": menu, "Conditions": "true"} self.call_contract("NewMenu", data) rollc_after = utils.getMaxIdFromTable(dbHost, dbName, login, pas, "rollback_tx") # change menu id = funcs.get_object_id(url, name, "menu", token) newValueMenu = menu.replace("new menu", "new_var") data = {"Id": id, "Value": newValueMenu} self.call_contract("EditMenu", data) # test query = """SELECT id FROM "rollback_tx" WHERE table_name = '1_menu' AND data='' AND id >= %s AND id <= %s""" % ( rollc_before, rollc_after) rollback_id = utils.executeSQL(dbHost, dbName, login, pas, query)[0][0] data = {"Table": "menu", "ID": id, "rID": rollback_id} contract = self.contracts["getHistoryRow"] self.check_contract_with_data(contract["code"], data, menu)
def form_valid(self, form): name = form.cleaned_data['name'] file = form.cleaned_data['file'] try: origin_name, ext = os.path.splitext(file.name) aws_name = 'pinna_ads/'\ + utils.generate_random_name(origin_name) + ext url = utils.add_object_via_cloudfront(aws_name, file, False) models.PinnaAd.objects.create(name=name, url=url, s3_name=aws_name) messages.add_message(self.request, messages.INFO, "Created new AD successfully") except Exception as e: LOG.error(traceback.format_exc()) messages.add_message(self.request, messages.ERROR, "Can't create AD: %s" %e) return super(AdCreate, self).form_valid(form)
def test_get_avatar_without_login(self): # add file in binaries name = "file_" + utils.generate_random_name() path = os.path.join(os.getcwd(), "fixtures", "image2.jpg") with open(path, 'rb') as f: file = f.read() files = {'Data': file} data = {"Name": name, "ApplicationId": 1, "DataMimeType": "image/jpeg"} resp = utils.call_contract_with_files(url, prKey, "UploadBinary", data, files, token) res = self.assertTxInBlock(resp, token) self.assertGreater(int(res), 0, "BlockId is not generated: " + res) # find last added file asserts = ["count"] res = self.check_get_api("/list/binaries", "", asserts) lastRec = res["count"] # find founder ID asserts = ["list"] res = self.check_get_api("/list/members", "", asserts) # iterating response elements i = 0 founderID = "" while i < len(res['list']): if res['list'][i]['member_name'] == "founder": founderID = res['list'][i]['id'] i += 1 # change column permissions data = { "TableName": "members", "Name": "image_id", "UpdatePerm": "true", "ReadPerm": "true" } res = self.call("EditColumn", data) self.assertGreater(int(res), 0, "BlockId is not generated: " + res) # update members table code = """ { data{} conditions{} action{ DBUpdate("members", %s, {image_id: "%s"}) } } """ % (founderID, lastRec) code, name = utils.generate_name_and_code(code) data = {"Value": code, "ApplicationId": 1, "Conditions": "true"} res = self.call("NewContract", data) self.assertGreater(int(res), 0, "BlockId is not generated: " + res) data = {} resp = utils.call_contract(url, prKey, name, data, token) res = self.assertTxInBlock(resp, token) self.assertGreater(int(res), 0, "BlockId is not generated: " + res) # rollback changes column permissions data = { "TableName": "members", "Name": "image_id", "UpdatePerm": "ContractAccess(\"Profile_Edit\")", "ReadPerm": "true" } res = self.call("EditColumn", data) self.assertGreater(int(res), 0, "BlockId is not generated: " + res) # test ecosystemID = "1" avaURL = url + "/avatar/" + ecosystemID + "/" + founderID resp = requests.get(avaURL) msg = "Content-Length is different!" self.assertIn("71926", str(resp.headers["Content-Length"]), msg)
def new_parameter(self): name = "Par_" + utils.generate_random_name() data = {"Name": name, "Value": "test", "Conditions": "true"} res = self.call("NewParameter", data) return name
def create_ecosystem(self): data = {"Name": "Ecosys" + utils.generate_random_name()} res = self.call("NewEcosystem", data)
def generate_cloud_service_name(self, os_user=None, random=False): if random: return utils.generate_random_name(10) return '-'.join((os_user, utils.generate_random_name(6)))
def test_get_ecosystem_name_new(self): data = {"Name": "ecos_" + utils.generate_random_name()} res = self.call("NewEcosystem", data) id = self.check_get_api("/list/ecosystems", "", [])["count"] asserts = ["ecosystem_name"] self.check_get_api("/ecosystemname?id=" + str(id), "", asserts)
def new_menu(self, url, prKey): name = "Menu_" + utils.generate_random_name() data = {"Name": name, "Value": "Item1", "Conditions": "true"} res = utils.call_contract(url, prKey, "NewMenu", data, self.data1["jwtToken"]) return funcs.get_count(url, "menu", self.data1["jwtToken"])
def generate_name_and_code(self, sourseCode): name = utils.generate_random_name() code = "contract " + name + sourseCode return code, name