def to_entity(self) -> dict: """ Convert a MagRelease instance into an Azure Table Storage entity. :return: an Azure Table Storage entity. """ entity = dict() entity[MagRelease.__PARTITION_KEY] = self.partition_key entity[MagRelease.__ROW_KEY] = self.row_key entity[MagRelease.__STATE] = self.state.value entity[MagRelease.__TASK] = self.task.value entity[MagRelease.__RELEASE_DATE] = EntityProperty( EdmType.DATETIME, value=self.release_date) entity[MagRelease.__SOURCE_CONTAINER] = self.source_container entity[MagRelease.__SOURCE_CONTAINER_LAST_MODIFIED] = \ EntityProperty(EdmType.DATETIME, value=self.source_container_last_modified) entity[MagRelease.__RELEASE_CONTAINER] = self.release_container entity[MagRelease.__RELEASE_PATH] = self.release_path entity[MagRelease.__DISCOVERED_DATE] = EntityProperty( EdmType.DATETIME, value=self.discovered_date) entity[MagRelease.__ARCHIVED_DATE] = EntityProperty( EdmType.DATETIME, value=self.archived_date) entity[MagRelease.__DONE_DATE] = EntityProperty(EdmType.DATETIME, value=self.done_date) return entity
def addingEntity(): i = 0 for t in time: bitcoin = {'PartitionKey': Bitcoinz, 'RowKey': str(t), 'Open': EntityProperty(EdmType.DOUBLE, openz[i]), 'Close': EntityProperty(EdmType.DOUBLE, close[i]), 'High': EntityProperty(EdmType.DOUBLE, high[i]), 'Low': EntityProperty(EdmType.DOUBLE, low[i]), 'Time': timez[i], 'priority': 200} bitcoins.append(bitcoin) if(len(bitcoins) > 99): sendEntities() i += 1
def azureTable(): # table_service = TableService(account_name='', account_key='') table_service = TableService( connection_string='') #table_service.create_table('bitcoinData') i = 0 for t in time: bitcoin = {'PartitionKey': Bitcoinz, 'RowKey': str(t), 'Open': EntityProperty(EdmType.DOUBLE, openz[i]), 'Close': EntityProperty(EdmType.DOUBLE, close[i]), 'High': EntityProperty(EdmType.DOUBLE, high[i]), 'Low': EntityProperty(EdmType.DOUBLE, low[i]), 'Time': timez[i], 'priority': 200} table_service.insert_entity('bitcoinData', bitcoin) i += 1
def updatetable(currency, day, rbf, linear, poly, accuracy): print("Uploading: " + str(currency) + str(day) + str(rbf) + str(linear) + str(poly) + str(accuracy)) table_service = TableService( connection_string=tablestorageconnectionstring) entity = { 'PartitionKey': currency, 'RowKey': day, 'RBF': EntityProperty(EdmType.DOUBLE, rbf), 'Linear': EntityProperty(EdmType.DOUBLE, linear), 'Polynomial': EntityProperty(EdmType.DOUBLE, poly), 'Accuracy': EntityProperty(EdmType.DOUBLE, accuracy) } table_service.insert_or_replace_entity('PredictionConfiguration', entity)
def _update_table_(df, service, az_config): logging.info(f'Starting write to storage table {az_config.OUTPUT_TABLE}.') batch_counter = 0 batch = TableBatch() for idx, row in df.iterrows(): task = { 'PartitionKey': row['PartitionKey'], 'RowKey': str(row['RowKey']), 'json_data': EntityProperty(EdmType.STRING, row['json_data']) } batch_counter += 1 batch.insert_entity(task) if batch_counter == 100: batch_counter = 0 service.commit_batch(az_config.OUTPUT_TABLE, batch) batch = None batch = TableBatch() if batch_counter > 0: service.commit_batch(az_config.OUTPUT_TABLE, batch) logging.info('Finished write to storage table.')
def update_table(currency, day, rbf, linear, poly, actual): print("Updating table: ", "\nCurrency: ", currency, "\nDay: ", day, "\nRBF: ", rbf, "\nLinear: ", linear, "\nPolynomial: ", poly, "\nActual: ", actual) table_service = TableService( connection_string=tablestorageconnectionstring) entity = { 'PartitionKey': currency, 'RowKey': day, 'RBF': EntityProperty(EdmType.DOUBLE, rbf), 'Linear': EntityProperty(EdmType.DOUBLE, linear), 'Polynomial': EntityProperty(EdmType.DOUBLE, poly), 'Actual': EntityProperty(EdmType.DOUBLE, actual) } table_service.insert_or_replace_entity('PredictionData', entity)
def post(self, mail): ''' Adds a message ''' if request.is_json: try: try: table_service.get_entity('users', mail, '') message = Entity() details = message_pb2.Message() details.title = request.json["title"] details.content = request.json["content"] details.magic_number = request.json["magic_number"] message.PartitionKey = mail message.RowKey = str( len(list(table_service.query_entities('messages'))) + 1) message.details = EntityProperty(EdmType.BINARY, MessageToJson(details)) table_service.insert_entity('messages', message) return None, 201 except (Azure404): return None, 404 except (KeyError): return 'Please provide a json object conforming to the \ following pattern: {\"title\": \"Message title\", \ \"content\":\"Message content\", \ \"magic_number\": a number}', 400
def add_entity(currency, day, rbf, linear, poly, actual, comboV1): print("Added Entity: ", "\nCurrency: ", currency, "\nDay: ", day, "\nRBF: ", rbf, "\nLinear: ", linear, "\nPolynomial: ", poly, "\nActual: ", actual, "\nComboV1: ", comboV1) entity = { 'PartitionKey': currency, 'RowKey': day, 'RBF': EntityProperty(EdmType.DOUBLE, rbf), 'Linear': EntityProperty(EdmType.DOUBLE, linear), 'Polynomial': EntityProperty(EdmType.DOUBLE, poly), 'Actual': EntityProperty(EdmType.DOUBLE, actual), 'ComboV1': EntityProperty(EdmType.DOUBLE, comboV1) } entities.append(entity) if (len(entities) > 99): uploadentities()
def get_next_region_name(): # Number of regions available for deployment (in the table). num_regions = table_service.get_entity( table_name, part_table_control, "NumberOfRegions" ).NumberOfRegions.value logging.debug("num_regions: {}".format(num_regions)) # Rowkey for the last region used (from the table). last_region_rowkey_used = get_last_region_rowkey_used() logging.debug("last_region_rowkey_used: {}".format(last_region_rowkey_used)) # Use the next region in the table. If the last region used was the last # region in the table, then this time use the first region in the table. curr_region_rowkey = last_region_rowkey_used + 1 if curr_region_rowkey > num_regions: curr_region_rowkey = 1 logging.debug("curr_region_rowkey: {}".format(curr_region_rowkey)) # Cooldown. Select regions whose last successful run was > 48 hours ago # or haven't had a last successful run. orig_rowkey = curr_region_rowkey last_rowkey = curr_region_rowkey - 1 if last_rowkey < 1: last_rowkey = num_regions logging.debug("last_rowkey = {}".format(last_rowkey)) while (hours_since_last_success(curr_region_rowkey) < cooldown_hours and curr_region_rowkey != last_rowkey): logging.debug("INFO: {0:.2f} hours since last successful run in region {1}. Skipping.".format( hours_since_last_success(curr_region_rowkey), get_region_shortname(curr_region_rowkey) )) curr_region_rowkey += 1 if curr_region_rowkey > num_regions: curr_region_rowkey = 1 if curr_region_rowkey == last_rowkey: curr_region_rowkey = orig_rowkey logging.debug("curr_region_rowkey (after cooldown): {}".format( curr_region_rowkey)) # Get the region shortname. region = get_region_shortname(curr_region_rowkey) logging.debug("region: {}".format(region)) # Update LastRegionRowKeyUsed. table_service.update_entity( table_name, { "PartitionKey": part_table_control, "RowKey": "LastRegionRowKeyUsed", "LastRegionRowKeyUsed": EntityProperty(EdmType.INT32, curr_region_rowkey) }) return region
def azureTable(): # table_service = TableService(account_name='', account_key='') table_service = TableService( connection_string= 'DefaultEndpointsProtocol=https;AccountName=sauokgp;AccountKey=113mdwUqIiqt4K2HonK80HakIOplxYZINmQME5KB1IZfP+v3JHZK64wpoTP5NBFaG0MaO/TVqA0nW4KuCINTow==;EndpointSuffix=core.windows.net' ) #table_service.create_table('bitcoinData') i = 0 for t in time: bitcoin = { 'PartitionKey': Bitcoinz, 'RowKey': str(t), 'Open': EntityProperty(EdmType.DOUBLE, openz[i]), 'Close': EntityProperty(EdmType.DOUBLE, close[i]), 'High': EntityProperty(EdmType.DOUBLE, high[i]), 'Low': EntityProperty(EdmType.DOUBLE, low[i]), 'Time': timez[i], 'priority': 200 } table_service.insert_entity('bitcoinData', bitcoin) i += 1
def post(self): """ Inserts user into an Azure table. """ if request.is_json: try: user = Entity() user.PartitionKey = request.json["email"] user.RowKey = '' user.info = EntityProperty(EdmType.BINARY, dumps(request.json)) table_service.insert_or_replace_entity('users', user) except (KeyError): return 'Please provide a json object conforming to \ the following pattern: {\"email\":\"[email protected]\",\ \"password\":\"xxx\", \"full_name\":\"Foo Bar\"}', 400 return None, 201 else: return 'Please supply a json object in order to add a user.', 400
def _from_entity_int32(value): return EntityProperty(EdmType.INT32, int(value))
def _from_entity_binary(value): return EntityProperty(EdmType.BINARY, _decode_base64_to_bytes(value))
def _convert_json_to_entity(entry_element, property_resolver, encrypted_properties): ''' Convert json response to entity. The entity format is: { "Address":"Mountain View", "Age":23, "AmountDue":200.23, "*****@*****.**":"Edm.Guid", "CustomerCode":"c9da6455-213d-42c9-9a79-3e9149a57833", "*****@*****.**":"Edm.DateTime", "CustomerSince":"2008-07-10T00:00:00", "IsActive":true, "*****@*****.**":"Edm.Int64", "NumberOfOrders":"255", "PartitionKey":"mypartitionkey", "RowKey":"myrowkey" } ''' entity = Entity() properties = {} edmtypes = {} odata = {} for name, value in entry_element.items(): if name.startswith('odata.'): odata[name[6:]] = value elif name.endswith('@odata.type'): edmtypes[name[:-11]] = value else: properties[name] = value # Partition key is a known property partition_key = properties.pop('PartitionKey', None) if partition_key: entity['PartitionKey'] = partition_key # Row key is a known property row_key = properties.pop('RowKey', None) if row_key: entity['RowKey'] = row_key # Timestamp is a known property timestamp = properties.pop('Timestamp', None) if timestamp: entity['Timestamp'] = _from_entity_datetime(timestamp) for name, value in properties.items(): mtype = edmtypes.get(name) # use the property resolver if present if property_resolver: # Clients are not expected to resolve these interal fields. # This check avoids unexpected behavior from the user-defined # property resolver. if not (name == '_ClientEncryptionMetadata1' or name == '_ClientEncryptionMetadata2'): mtype = property_resolver(partition_key, row_key, name, value, mtype) # throw if the type returned is not a valid edm type if mtype and mtype not in _EDM_TYPES: raise AzureException( _ERROR_TYPE_NOT_SUPPORTED.format(mtype)) # If the property was encrypted, supercede the results of the resolver and set as binary if encrypted_properties is not None and name in encrypted_properties: mtype = EdmType.BINARY # Add type for Int32 if type(value) is int: mtype = EdmType.INT32 # no type info, property should parse automatically if not mtype: entity[name] = value else: # need an object to hold the property conv = _ENTITY_TO_PYTHON_CONVERSIONS.get(mtype) if conv is not None: try: property = conv(value) except Exception as e: # throw if the type returned by the property resolver # cannot be used in the conversion if property_resolver: raise AzureException( _ERROR_INVALID_PROPERTY_RESOLVER.format( name, value, mtype)) else: raise e else: property = EntityProperty(mtype, value) entity[name] = property # extract etag from entry etag = odata.get('etag') if timestamp: etag = 'W/"datetime\'' + url_quote(timestamp) + '\'"' entity['etag'] = etag return entity
def _encrypt_entity(entity, key_encryption_key, encryption_resolver): ''' Encrypts the given entity using AES256 in CBC mode with 128 bit padding. Will generate a content-encryption-key (cek) to encrypt the properties either stored in an EntityProperty with the 'encrypt' flag set or those specified by the encryption resolver. This cek is then wrapped using the provided key_encryption_key (kek). Only strings may be encrypted and the result is stored as binary on the service. :param entity: The entity to insert. Could be a dict or an entity object. :param object key_encryption_key: The user-provided key-encryption-key. Must implement the following methods: wrap_key(key)--wraps the specified key using an algorithm of the user's choice. get_key_wrap_algorithm()--returns the algorithm used to wrap the specified symmetric key. get_kid()--returns a string key id for this key-encryption-key. :param function(partition_key, row_key, property_name) encryption_resolver: A function that takes in an entities partition key, row key, and property name and returns a boolean that indicates whether that property should be encrypted. :return: An entity with both the appropriate properties encrypted and the encryption data. :rtype: object ''' _validate_not_none('entity', entity) _validate_not_none('key_encryption_key', key_encryption_key) _validate_key_encryption_key_wrap(key_encryption_key) # AES256 uses 256 bit (32 byte) keys and always with 16 byte blocks content_encryption_key = os.urandom(32) entity_initialization_vector = os.urandom(16) encrypted_properties = [] encrypted_entity = Entity() for key, value in entity.items(): # If the property resolver says it should be encrypted # or it is an EntityProperty with the 'encrypt' property set. if (isinstance(value, EntityProperty) and value.encrypt) or \ (encryption_resolver is not None \ and encryption_resolver(entity['PartitionKey'], entity['RowKey'], key)): # Only strings can be encrypted and None is not an instance of str. if isinstance(value, EntityProperty): if value.type == EdmType.STRING: value = value.value else: raise ValueError(_ERROR_UNSUPPORTED_TYPE_FOR_ENCRYPTION) if not isinstance(value, str): raise ValueError(_ERROR_UNSUPPORTED_TYPE_FOR_ENCRYPTION) # Value is now confirmed to hold a valid string value to be encrypted # and should be added to the list of encrypted properties. encrypted_properties.append(key) propertyIV = _generate_property_iv(entity_initialization_vector, entity['PartitionKey'], entity['RowKey'], key, False) # Encode the strings for encryption. value = value.encode('utf-8') cipher = _generate_AES_CBC_cipher(content_encryption_key, propertyIV) # PKCS7 with 16 byte blocks ensures compatibility with AES. padder = PKCS7(128).padder() padded_data = padder.update(value) + padder.finalize() # Encrypt the data. encryptor = cipher.encryptor() encrypted_data = encryptor.update( padded_data) + encryptor.finalize() # Set the new value of this key to be a binary EntityProperty for proper serialization. value = EntityProperty(EdmType.BINARY, encrypted_data) encrypted_entity[key] = value encrypted_properties = dumps(encrypted_properties) # Generate the metadata iv. metadataIV = _generate_property_iv(entity_initialization_vector, entity['PartitionKey'], entity['RowKey'], '_ClientEncryptionMetadata2', False) encrypted_properties = encrypted_properties.encode('utf-8') cipher = _generate_AES_CBC_cipher(content_encryption_key, metadataIV) padder = PKCS7(128).padder() padded_data = padder.update(encrypted_properties) + padder.finalize() encryptor = cipher.encryptor() encrypted_data = encryptor.update(padded_data) + encryptor.finalize() encrypted_entity['_ClientEncryptionMetadata2'] = EntityProperty( EdmType.BINARY, encrypted_data) encryption_data = _generate_encryption_data_dict( key_encryption_key, content_encryption_key, entity_initialization_vector) encrypted_entity['_ClientEncryptionMetadata1'] = dumps(encryption_data) return encrypted_entity
def update_available_region_names(): # Authenticate to Azure Subscription Manager sub_client = SubscriptionClient( ServicePrincipalCredentials(client_id=os.environ["AZURE_CLIENT_ID"], secret=os.environ["AZURE_CLIENT_SECRET"], tenant=os.environ["AZURE_TENANT_ID"])) # Get the list of locations accessible to this subscription. sub_id = os.environ["AZURE_SUBSCRIPTION_ID"] locations = sorted( [x.name for x in sub_client.subscriptions.list_locations(sub_id)] + regions_to_include) locations = [x for x in locations if x not in regions_to_exclude] # Get entities for all of the regions. old_entities = table_service.query_entities( table_name, filter="PartitionKey eq '{}'".format(part_deploy_region)) old_entities_dict = {x.RegionShortname: x for x in old_entities} # Create a new list of entities for the table. new_entities = [] for i, loc in enumerate(locations): # Remove this region from the original list and table. old_entity = old_entities_dict.pop(loc, {}) # Save the last successful run timestamp (if it exists). last_succ_run = None if "LastSuccessfulRun" in old_entity: last_succ_run = old_entity.LastSuccessfulRun # Create a new entity for this region (re-indexed). new_entities.append( Entity(PartitionKey=part_deploy_region, RowKey=str(i + 1), RegionShortname=loc, LastSuccessfulRun=last_succ_run)) logging.debug("new_entities = {}".format(new_entities)) # If there are any entries left in old_entities_dict, then that means # some regions that were live before are no longer live. if len(old_entities_dict): logging.debug( "WARNING: The following regions seem to have been decomissioned:") for k, v in old_entities_dict.items(): logging.debug("\t{}".format(k)) # Delete the old entities. for oe in old_entities: table_service.delete_entity(table_name, oe.PartitionKey, oe.RowKey) # Add the new entities. for ne in new_entities: table_service.insert_entity(table_name, ne) # Update the last region RowKey (the number of new entities). last_rowkey_entity = table_service.get_entity(table_name, part_table_control, "NumberOfRegions") last_rowkey_entity.pop("etag", None) last_rowkey_entity.pop("Timestamp", None) last_rowkey_entity.VALUE = EntityProperty(EdmType.INT32, len(new_entities)) logging.debug("last_rowkey_entity = {}".format(last_rowkey_entity)) table_service.update_entity(table_name, last_rowkey_entity)
'device_id=e4ba4977314e200234880bac6b3585af; s=f61bvmhzj7; _ga=GA1.2.1562254249.1522590556; _gid=GA1.2.447715148.1528814975; xq_a_token=019174f18bf425d22c8e965e48243d9fcfbd2cc0; xq_a_token.sig=_pB0kKy3fV9fvtvkOzxduQTrp7E; xq_r_token=2d465aa5d312fbe8d88b4e7de81e1e915de7989a; xq_r_token.sig=lOCElS5ycgbih9P-Ny3cohQ-FSA; Hm_lvt_1db88642e346389874251b5a1eded6e3=1528814975,1528897343; Hm_lpvt_1db88642e346389874251b5a1eded6e3=1528897343; u=111528897342585; _gat_gtag_UA_16079156_4=1' } req = urllib.request.Request(url, headers=headers) response = urllib.request.urlopen(req) jdata = json.loads(response.read()) if self.total_symbols_to_crawl == 0: self.total_symbols_to_crawl = int(jdata['count']['count']) for eachStock in jdata['stocks']: self.stocks.append( Stock(eachStock['symbol'], eachStock['code'], eachStock['name'])) self.total_symbols_to_crawled += 1 print('current parsed to page {}, {} symbols queried'.format( self.current_page, self.total_symbols_to_crawled)) if __name__ == '__main__': crawler = SymbolCrawler() while crawler.has_next(): crawler.crawl_next() table_service = TableService(account_name='heaventextb06a', account_key='**fill your own key**') for each_stock in crawler.stocks: print(each_stock) body = EntityProperty(EdmType.STRING, str(each_stock)) table_service.insert_or_replace_entity('heavenstock', each_stock)