def test_batch_merge(self): # Arrange # Act entity = Entity() entity.PartitionKey = '001' entity.RowKey = 'batch_merge' entity.test = EntityProperty(EdmType.BOOLEAN, 'true') entity.test2 = 'value' entity.test3 = 3 entity.test4 = EntityProperty(EdmType.INT64, '1234567890') entity.test5 = datetime.utcnow() self.ts.insert_entity(self.table_name, entity) entity = self.ts.get_entity(self.table_name, '001', 'batch_merge') self.assertEqual(3, entity.test3) entity = Entity() entity.PartitionKey = '001' entity.RowKey = 'batch_merge' entity.test2 = 'value1' batch = TableBatch() batch.merge_entity(entity) resp = self.ts.commit_batch(self.table_name, batch) # Assert self.assertIsNotNone(resp) entity = self.ts.get_entity(self.table_name, '001', 'batch_merge') self.assertEqual('value1', entity.test2) self.assertEqual(1234567890, entity.test4) self.assertEqual(resp[0], entity.etag)
def _create_random_entity_class(self, pk=None, rk=None): ''' Creates a class-based entity with fixed values, using all of the supported data types. ''' partition = pk if pk is not None else self.get_resource_name('pk') row = rk if rk is not None else self.get_resource_name('rk') entity = Entity() entity.PartitionKey = partition entity.RowKey = row entity.age = 39 entity.sex = 'male' entity.married = True entity.deceased = False entity.optional = None entity.evenratio = 3.0 entity.ratio = 3.1 entity.large = 933311100 entity.Birthday = datetime(1973, 10, 4) entity.birthday = datetime(1970, 10, 4) entity.binary = None entity.other = EntityProperty(EdmType.INT32, 20) entity.clsid = EntityProperty(EdmType.GUID, 'c9da6455-213d-42c9-9a79-3e9149a57833') return entity
def test_query_entities_large(self): # Arrange table_name = self._create_query_table(0) total_entities_count = 1000 entities_per_batch = 50 for j in range(total_entities_count // entities_per_batch): batch = TableBatch() for i in range(entities_per_batch): entity = Entity() entity.PartitionKey = 'large' entity.RowKey = 'batch{0}-item{1}'.format(j, i) entity.test = EntityProperty(EdmType.BOOLEAN, 'true') entity.test2 = 'hello world;' * 100 entity.test3 = 3 entity.test4 = EntityProperty(EdmType.INT64, '1234567890') entity.test5 = datetime(2016, 12, 31, 11, 59, 59, 0) batch.insert_entity(entity) self.ts.commit_batch(table_name, batch) # Act start_time = datetime.now() entities = list(self.ts.query_entities(table_name)) elapsed_time = datetime.now() - start_time # Assert print('query_entities took {0} secs.'.format( elapsed_time.total_seconds())) # azure allocates 5 seconds to execute a query # if it runs slowly, it will return fewer results and make the test fail self.assertEqual(len(entities), total_entities_count)
def insert_or_replace_entity_from_pi_azure_raw_accel(table_service, rowKey, entry, table_name='test', partitionKey='default'): ''' takes table service Takes a list Uploads to azure table storage ''' segment = Entity() segment.PartitionKey = partitionKey segment.RowKey = str(rowKey).zfill(8) segment.latA = str(entry['latA']) segment.longA = str(entry['longA']) segment.latB = str(entry['latB']) segment.longB = str(entry['longB']) segment.colorKey = str(entry['color']) if entry['x']: segment.x = str(entry['x']) segment.y = str(entry['y']) segment.z = str(entry['z']) print segment table_service.insert_or_replace_entity(table_name, segment)
def vehicleToEntity(dic, partitionKey, dump = False): entity = Entity() entity.PartitionKey = partitionKey entity.RowKey = str(dic['timestamp']) entity.camera = dic['camera']['id'] entity.camera_activation = str(dic['camera']['timestamp']) entity.street = dic['camera']['street'] entity.city = dic['camera']['city'] entity.speedLimit = dic['camera']['speedLimit'] entity.plate = dic['vehicle']['plate'] entity.type = dic['vehicle']['type'] entity.speed = dic['vehicle']['speed'] entity.isSpeeding = dic['vehicle']['isSpeeding'] entity.isPriority = dic['vehicle']['isPriority'] if dump: print "" print "Partition Key = %s" % entity.PartitionKey print "Row Key = %s" % entity.RowKey print "Camera:" print " Id = %s" % entity.camera print " Street = %s" % entity.street print " City = %s" % entity.city print " Speed Limit = %d" % entity.speedLimit print " Last Activation = %s" % entity.camera_activation print "Vehicle:" print " Plate = %s" % entity.plate print " Type = %s" % entity.type print " Speed = %s" % str(entity.speed) print " Speeding = %s" % entity.isSpeeding print " Priority = %s" % entity.isPriority print "" return entity
def post(self): res = {} try: # get the posted user data userdata = request.data # save to blob blob_file_name = self.partition_name + '.json' block_blob_service.create_blob_from_text(self.container, blob_file_name, userdata) #save to table task = Entity() task.PartitionKey = self.partition_name userModel = json.loads(userdata) task.RowKey = userModel['userName'] task.FirstName = userModel['firstName'] task.LastName = userModel['lastName'] task.EmailAddress = userModel['emailAddress'] task.CellNo = userModel['cellNo'] table_service.insert_entity(tablename, task) res = {'message': 'save successful'} except expression as identifier: res = {'message': 'save failed'} return jsonify(res)
def insert_or_replace_entity_to_azure(self, rowKey, entry, t_name=DEFAULT_TABLE): ''' takes table service Takes a list Uploads to azure table storage ''' segment = Entity() segment.PartitionKey = self.default_partition segment.RowKey = str(rowKey).zfill(8) segment.latA = str(entry['latA']) segment.longA = str(entry['longA']) segment.latB = str(entry['latB']) segment.longB = str(entry['longB']) segment.colorKey = str(entry['color']) #print segment.colorKey if os.name == 'nt': self.table_service.insert_or_replace_entity( t_name, self.default_partition, str(rowKey).zfill(8), segment) else: self.table_service.insert_or_replace_entity(t_name, segment)
def importALotOfRows(): for a in range(1,1100): task = Entity() task.PartitionKey = 'participant' task.RowKey = "participant_" + str(a) task.userID = a task.datetime = datetime.now().strftime("%Y-%m-%d %H:%M:%S") task.version = a table_service.insert_or_replace_entity('participants', task)
def UpdateState(self, newstate): logging.info("Execution status update: " + newstate) self.State = newstate execrec = Entity() execrec.PartitionKey = self.Command.RowKey execrec.RowKey = self.Command.PartitionKey execrec.Executable = self.Command.CommandLine execrec.State = self.State execrec.Output = self.Output temp = tablesvc.insert_or_replace_entity(Tables.ExecTable, execrec)
def _create_random_base_entity_class(self): ''' Creates a class-based entity with only pk and rk. ''' partition = self.get_resource_name('pk') row = self.get_resource_name('rk') entity = Entity() entity.PartitionKey = partition entity.RowKey = row return entity
async def go(): entity = Entity() entity.Foo = 'Foo' entity.Ufo = 'Ufo' entity.Number = 1 entity.PartitionKey = 'app' entity.RowKey = '2' data = await self.bs.insert_entity('versions', entity) self.assertIsNotNone(data)
def test_batch_reuse(self): # Arrange table2 = self._get_table_reference('table2') self.ts.create_table(table2) # Act entity = Entity() entity.PartitionKey = '003' entity.RowKey = 'batch_all_operations_together-1' entity.test = EntityProperty(EdmType.BOOLEAN, 'true') entity.test2 = 'value' entity.test3 = 3 entity.test4 = EntityProperty(EdmType.INT64, '1234567890') entity.test5 = datetime.utcnow() batch = TableBatch() batch.insert_entity(entity) entity.RowKey = 'batch_all_operations_together-2' batch.insert_entity(entity) entity.RowKey = 'batch_all_operations_together-3' batch.insert_entity(entity) entity.RowKey = 'batch_all_operations_together-4' batch.insert_entity(entity) self.ts.commit_batch(self.table_name, batch) self.ts.commit_batch(table2, batch) batch = TableBatch() entity.RowKey = 'batch_all_operations_together' batch.insert_entity(entity) entity.RowKey = 'batch_all_operations_together-1' batch.delete_entity(entity.PartitionKey, entity.RowKey) entity.RowKey = 'batch_all_operations_together-2' entity.test3 = 10 batch.update_entity(entity) entity.RowKey = 'batch_all_operations_together-3' entity.test3 = 100 batch.merge_entity(entity) entity.RowKey = 'batch_all_operations_together-4' entity.test3 = 10 batch.insert_or_replace_entity(entity) entity.RowKey = 'batch_all_operations_together-5' batch.insert_or_merge_entity(entity) self.ts.commit_batch(self.table_name, batch) resp = self.ts.commit_batch(table2, batch) # Assert self.assertEqual(6, len(resp)) entities = list( self.ts.query_entities(self.table_name, "PartitionKey eq '003'", '')) self.assertEqual(5, len(entities))
def addCoffee(rowKey, brand, flavor, size, price): coffee = Entity() coffee.PartitionKey = 'coffeeshop' coffee.RowKey = str(rowKey) coffee.brand = brand coffee.flavor = flavor coffee.size = size coffee.price = price table_service.insert_entity('itemstable', coffee) print('Created entry for coffee ' + brand + ' ' + flavor) return
def addCar(rowKey, make, model, year, color, price): car = Entity() car.PartitionKey = 'cardealership' car.RowKey = str(rowKey) car.make = make car.model = model car.year = year car.color = color car.price = price table_service.insert_entity('itemstable', car) print('Created entry for car ' + make + ' ' + model) return
def UpdateState(self, newstate): #update the status of this node self.State = newstate logging.info('STATUS CHANGE: ' + self.State) status = Entity() status.PartitionKey = self.Pool status.RowKey = self.Name status.Command = "" status.State = self.State status.IP = self.IP status.OS = self.OS status.Size = self.Size temp = tablesvc.insert_or_replace_entity(Tables.NodeTable, status)
def put(self): userModel = json.loads(request.data) userName = str(userModel['userName']) #save to table task = Entity() task.PartitionKey = 'f6dcfcfe-169e-4731-a589-2f8c6c75768a' #self.partition_name task.RowKey = userModel['userName'] task.FirstName = userModel['firstName'] task.LastName = userModel['lastName'] task.EmailAddress = userModel['emailAddress'] task.CellNo = userModel['cellNo'] table_service.update_entity(tablename, task, if_match='*')
def test_batch_too_many_ops(self): # Arrange entity = self._create_default_entity_dict('001', 'batch_negative_1') self.ts.insert_entity(self.table_name, entity) # Act with self.assertRaises(AzureBatchValidationError): batch = TableBatch() for i in range(0, 101): entity = Entity() entity.PartitionKey = 'large' entity.RowKey = 'item{0}'.format(i) batch.insert_entity(entity) self.ts.commit_batch(self.table_name, batch)
def addNewParticipant(id, address, ownReferralLink, isConfirmed, isCompleted = False, invitedReferral = ''): try: task = Entity() task.PartitionKey = 'participant' task.RowKey = "participant_" + str(id) task.userID = id task.ethereumAddress = address task.referral = ownReferralLink task.invitedReferralLink = invitedReferral task.isConfirmed = isConfirmed task.isCompleted = isCompleted task.datetime = datetime.now().strftime("%Y-%m-%d %H:%M:%S") task.version = 1 table_service.insert_or_replace_entity('participants', task) except Exception as e: logging.error(traceback.format_exc())
def entry_page(): if request.method == 'POST': json_dict = request.get_json(True)[1:] print('Lenght is '+ str(json_dict[0])) for dics in json_dict: data_out=Entity() data_out.PartitionKey = dics['username'] data_out.movieID = dics['movieId'] data_out.rate=dics["rate"] data_out.RowKey=dics["time"] toDelete=len('data:image/jpeg;base64,')-1 pic64=dics["picuri"][toDelete:] data=pic64.decode('base64') headers = dict() headers['Ocp-Apim-Subscription-Key'] = _key headers['Content-Type'] = 'application/octet-stream' json = None params = None resultAPI = processRequest( json, data, headers, params ) data_out.anger=resultAPI[0]['scores']['anger'] data_out.contempt=resultAPI[0]['scores']['contempt'] data_out.disgust=resultAPI[0]['scores']['disgust'] data_out.fear=resultAPI[0]['scores']['fear'] data_out.happiness=resultAPI[0]['scores']['happiness'] data_out.neutral=resultAPI[0]['scores']['neutral'] data_out.sadness=resultAPI[0]['scores']['sadness'] data_out.surprise=resultAPI[0]['scores']['surprise'] table_service.create_table('DataForML') table_service.insert_or_replace_entity('DataForML', data_out) return 'Uploaded' else: output = table_service.get_entity('DataForML',"Anson",12134) return 'Everything is working'
def test_batch_all_operations_together_context_manager(self): # Arrange # Act entity = Entity() entity.PartitionKey = '003' entity.RowKey = 'batch_all_operations_together-1' entity.test = EntityProperty(EdmType.BOOLEAN, 'true') entity.test2 = 'value' entity.test3 = 3 entity.test4 = EntityProperty(EdmType.INT64, '1234567890') entity.test5 = datetime.utcnow() self.ts.insert_entity(self.table_name, entity) entity.RowKey = 'batch_all_operations_together-2' self.ts.insert_entity(self.table_name, entity) entity.RowKey = 'batch_all_operations_together-3' self.ts.insert_entity(self.table_name, entity) entity.RowKey = 'batch_all_operations_together-4' self.ts.insert_entity(self.table_name, entity) with self.ts.batch(self.table_name) as batch: entity.RowKey = 'batch_all_operations_together' batch.insert_entity(entity) entity.RowKey = 'batch_all_operations_together-1' batch.delete_entity(entity.PartitionKey, entity.RowKey) entity.RowKey = 'batch_all_operations_together-2' entity.test3 = 10 batch.update_entity(entity) entity.RowKey = 'batch_all_operations_together-3' entity.test3 = 100 batch.merge_entity(entity) entity.RowKey = 'batch_all_operations_together-4' entity.test3 = 10 batch.insert_or_replace_entity(entity) entity.RowKey = 'batch_all_operations_together-5' batch.insert_or_merge_entity(entity) # Assert entities = list( self.ts.query_entities(self.table_name, "PartitionKey eq '003'", '')) self.assertEqual(5, len(entities))
def addReferralToMainUser(id, referralLink): try: task = Entity() task.PartitionKey = 'referral_' + referralLink task.RowKey = "referral_" + str(id) task.userID = id task.referral = referralLink task.datetime = datetime.now().strftime("%Y-%m-%d %H:%M:%S") task.version = 1 table_service.insert_or_replace_entity('participantReferrer', task) #update participant item = table_service.get_entity('participants', 'participant', "participant_" + str(id)) item.isConfirmed = True item.isCompleted = True item.invitedReferralLink = referralLink table_service.insert_or_replace_entity('participants', item) except Exception as e: logging.error(traceback.format_exc())
def insert_entry_to_azure(table_service, rowKey, entry, table_name='test', partitionKey='default'): ''' takes table service Takes a list Uploads to azure table storage ''' segment = Entity() segment.PartitionKey = partitionKey segment.RowKey = str(rowKey).zfill(8) segment.latA = str(entry['latA']) segment.longA = str(entry['longA']) segment.latB = str(entry['latB']) segment.longB = str(entry['longB']) segment.colorKey = str(entry['color']) print segment table_service.insert_entity(table_name, segment)
def dictToEntity(self, dic): entity = Entity() if dic['camera']['isActive'] == "True": entity.PartitionKey = self.PARTITION_ACTIVATION else: entity.PartitionKey = self.PARTITION_DEACTIVATION entity.RowKey = str(dic['camera']['timestamp']) entity.id = dic['camera']['id'] entity.street = dic['camera']['street'] entity.city = dic['camera']['city'] entity.speedLimit = dic['camera']['speedLimit'] entity.rate = dic['camera']['rate'] if self.dump: print "" print "Partition Key = %s" % entity.PartitionKey print "Row Key = %s" % entity.RowKey print "Camera Id = %s" % entity.id print "Street = %s" % entity.street print "City = %s" % entity.city print "Speed Limit = %d" % entity.speedLimit print "Rate = %d" % entity.rate print "" return entity
def test_batch_inserts(self): # Arrange # Act entity = Entity() entity.PartitionKey = 'batch_inserts' entity.test = EntityProperty(EdmType.BOOLEAN, 'true') entity.test2 = 'value' entity.test3 = 3 entity.test4 = EntityProperty(EdmType.INT64, '1234567890') batch = TableBatch() for i in range(100): entity.RowKey = str(i) batch.insert_entity(entity) self.ts.commit_batch(self.table_name, batch) entities = list( self.ts.query_entities(self.table_name, "PartitionKey eq 'batch_inserts'", '')) # Assert self.assertIsNotNone(entities) self.assertEqual(100, len(entities))
### # Use the Azure Storage Storage SDK for Python to create some entries in the Table ### print('Now let\'s add some entries to our Table.\nRemember, Azure Storage Tables is a NoSQL datastore, so this is similar to adding records to a database.') raw_input('Press Enter to continue...') # Each entry in a Table is called an 'Entity'. # Here, we add an entry for first car with two pieces of data - the name, and the cost # # A partition key tracks how like-minded entries in the Table are created and queried. # A row key is a unique ID for each entity in the partition # These two properties are used as a primary key to index the Table. This makes queries much quicker. car = Entity() car.PartitionKey = 'cars' car.RowKey = '001' car.make = 'Ford' car.model = 'Edge' car.year = '2007' car.color = 'Gray' car.price = 32000 table_service.insert_entity('itemstable', car) print('Created entry for 2007 Ford Edge...') car = Entity() car.PartitionKey = 'cars' car.RowKey = '002' car.make = 'GMC' car.model = 'Acadia'
### # Use the Azure Storage Storage SDK for Python to create some entries in the Table ### print( 'Now let\'s add some entries to our Table.\nRemember, Azure Storage Tables is a NoSQL datastore, so this is similar to adding records to a database.' ) input('Press Enter to continue...') # Each entry in a Table is called an 'Entity'. # Here, we add five entries for cars with five pieces of data: 1) make, 2) model, 3) year, 4) color, 5) price # # A partition key tracks how like-minded entries in the Table are created and queried. # A row key is a unique ID for each entity in the partition # These two properties are used as a primary key to index the Table. This makes queries much quicker. cars = Entity() cars.PartitionKey = 'car_selections' cars.RowKey = '001' cars.make = 'Volkswagen' cars.model = 'Jetta' cars.year = 2010 cars.color = 'silver' cars.price = 13300 table_service.insert_entity('htangitemstable', cars) print('Created entry for Volkswagen/Jetta...') cars = Entity() cars.PartitionKey = 'car_selections' cars.RowKey = '002' cars.make = 'Ford' cars.model = 'Focus'
### # Use the Azure Storage Storage SDK for Python to create some entries in the Table ### print('Now let\'s add some entries to our Table.\nRemember, Azure Storage Tables is a NoSQL datastore, so this is similar to adding records to a database.') raw_input('Press Enter to continue...') # Each entry in a Table is called an 'Entity'. # Here, we add an entry for first pizza with two pieces of data - the name, and the cost # # A partition key tracks how like-minded entries in the Table are created and queried. # A row key is a unique ID for each entity in the partition # These two properties are used as a primary key to index the Table. This makes queries much quicker. pizza = Entity() pizza.PartitionKey = 'pizzamenu' pizza.RowKey = '001' pizza.description = 'Pepperoni' pizza.cost = 18 table_service.insert_entity('itemstable', pizza) print('Created entry for pepperoni...') pizza = Entity() pizza.PartitionKey = 'pizzamenu' pizza.RowKey = '002' pizza.description = 'Veggie' pizza.cost = 15 table_service.insert_entity('itemstable', pizza) print('Created entry for veggie...')
def insert_options_azure(optionChain): # receives the optionChain object containing all options for all expiration dates # for the selected symbol. inserts rows into the database options table for # each option. Performs a db INSERT statement. If the row already exists, # the database will generate an invalid key error to prevent the row from # being duplicated in the table. In this case, the error is ignored. # account = ut.get_azure_account() table_service = None table_name = ut.TABLE_NAME_OPTIONS try: if config.IS_EMULATED: table_service = TableService(is_emulated=True) else: table_service = TableService( account_name=config.STORAGE_ACCOUNT_NAME, account_key=config.STORAGE_ACCOUNT_KEY) if not table_service.exists(table_name): # create the table try: table_service.create_table(table_name) except Exception as err: print('Error creating table, ' + table_name + 'check if it already exists') lg.error( 'Tried and failed to create the table for the symbols. Program terminating...' ) exit() batch = TableBatch() batchCount = 0 rowCount = 0 for o in optionChain.options: rowCount += 1 if rowCount > 100: # Azure restricts the batch size to a max of a hundred entries. Since we're at our # limit, we'll commit these and start a new batch table_service.commit_batch(table_name, batch) batch = TableBatch() rowCount = 1 batchCount += 1 option = Entity() option.PartitionKey = o.PartitionKey # rowkey comprises the concatination of symbols to ensure the key is unique for the symbol. # we'll use the callPut, optionDate, expirationDate, and strike price. Dates will be in format yyyymmdd option.RowKey = o.RowKey option.OptionDate = o.optionDate # dates are already cast as Entity Property with an aware date value option.Expiration = o.expiration option.CallPut = o.callPut option.Strike = o.strike option.Bid = o.bid option.Ask = o.ask option.LastPrice = o.lastPrice option.Volume = o.volume option.OpenInterest = o.openInterest option.IV = o.impliedVolatility option.StockPrice = o.stockPrice batch.insert_entity(option) table_service.commit_batch(table_name, batch) except Exception as e: print('Error adding option ' + symbol + '. Error is: ', e) lg.error('Error adding rows to the options table')
# pizza.description = 'Veggie' # pizza.cost = 15 # table_service.insert_entity('itemstable', pizza) # print('Created entry for veggie...') # pizza = Entity() # pizza.PartitionKey = 'pizzamenu' # pizza.RowKey = '003' # pizza.description = 'Hawaiian' # pizza.cost = 12 # table_service.insert_entity('itemstable', pizza) # print('Created entry for Hawaiian...\n') # Shailesh Beri - Add code here to populate partition with cars using dealership scenario and with characteristics [make, model, year, color and price] instead of Pizza cars = Entity() cars.PartitionKey = 'carstype' cars.RowKey = '001' cars.make = 'BMW' cars.model = 'X3' cars.year = 2017 cars.color = 'Red' cars.price = 51000 table_service.insert_entity('itemstable', cars) print('Created entry for BMW X3...') cars = Entity() cars.PartitionKey = 'carstype' cars.RowKey = '002' cars.make = 'Audi' cars.model = 'Q5'
### # Use the Azure Storage Storage SDK for Python to create some entries in the Table ### print( 'Now let\'s add some entries to our Table.\nRemember, Azure Storage Tables is a NoSQL datastore, so this is similar to adding records to a database.' ) raw_input('Press Enter to continue...') # Each entry in a Table is called an 'Entity'. # Here, we add an entry for first pizza with two pieces of data - the name, and the cost # # A partition key tracks how like-minded entries in the Table are created and queried. # A row key is a unique ID for each entity in the partition # These two properties are used as a primary key to index the Table. This makes queries much quicker. pizza = Entity() pizza.PartitionKey = 'pizzamenu' pizza.RowKey = '001' pizza.description = 'Pepperoni' pizza.cost = 18 table_service.insert_entity('itemstable', pizza) print('Created entry for pepperoni...') pizza = Entity() pizza.PartitionKey = 'pizzamenu' pizza.RowKey = '002' pizza.description = 'Veggie' pizza.cost = 15 table_service.insert_entity('itemstable', pizza) print('Created entry for veggie...')