def to_entity(self) -> dict:
""" Convert a MagRelease instance into an Azure Table Storage entity.
:return: an Azure Table Storage entity.
"""
entity = dict()
entity[MagRelease.__PARTITION_KEY] = self.partition_key
entity[MagRelease.__ROW_KEY] = self.row_key
entity[MagRelease.__STATE] = self.state.value
entity[MagRelease.__TASK] = self.task.value
entity[MagRelease.__RELEASE_DATE] = EntityProperty(
EdmType.DATETIME, value=self.release_date)
entity[MagRelease.__SOURCE_CONTAINER] = self.source_container
entity[MagRelease.__SOURCE_CONTAINER_LAST_MODIFIED] = \
EntityProperty(EdmType.DATETIME, value=self.source_container_last_modified)
entity[MagRelease.__RELEASE_CONTAINER] = self.release_container
entity[MagRelease.__RELEASE_PATH] = self.release_path
entity[MagRelease.__DISCOVERED_DATE] = EntityProperty(
EdmType.DATETIME, value=self.discovered_date)
entity[MagRelease.__ARCHIVED_DATE] = EntityProperty(
EdmType.DATETIME, value=self.archived_date)
entity[MagRelease.__DONE_DATE] = EntityProperty(EdmType.DATETIME,
value=self.done_date)
return entity
def addingEntity():
i = 0
for t in time:
bitcoin = {'PartitionKey': Bitcoinz, 'RowKey': str(t), 'Open': EntityProperty(EdmType.DOUBLE, openz[i]),
'Close': EntityProperty(EdmType.DOUBLE, close[i]), 'High': EntityProperty(EdmType.DOUBLE, high[i]),
'Low': EntityProperty(EdmType.DOUBLE, low[i]), 'Time': timez[i], 'priority': 200}
bitcoins.append(bitcoin)
if(len(bitcoins) > 99):
sendEntities()
i += 1
def azureTable():
# table_service = TableService(account_name='', account_key='')
table_service = TableService(
connection_string='')
#table_service.create_table('bitcoinData')
i = 0
for t in time:
bitcoin = {'PartitionKey': Bitcoinz, 'RowKey': str(t), 'Open': EntityProperty(EdmType.DOUBLE, openz[i]),
'Close': EntityProperty(EdmType.DOUBLE, close[i]), 'High': EntityProperty(EdmType.DOUBLE, high[i]),
'Low': EntityProperty(EdmType.DOUBLE, low[i]), 'Time': timez[i], 'priority': 200}
table_service.insert_entity('bitcoinData', bitcoin)
i += 1
def updatetable(currency, day, rbf, linear, poly, accuracy):
print("Uploading: " + str(currency) + str(day) + str(rbf) + str(linear) +
str(poly) + str(accuracy))
table_service = TableService(
connection_string=tablestorageconnectionstring)
entity = {
'PartitionKey': currency,
'RowKey': day,
'RBF': EntityProperty(EdmType.DOUBLE, rbf),
'Linear': EntityProperty(EdmType.DOUBLE, linear),
'Polynomial': EntityProperty(EdmType.DOUBLE, poly),
'Accuracy': EntityProperty(EdmType.DOUBLE, accuracy)
}
table_service.insert_or_replace_entity('PredictionConfiguration', entity)
def _update_table_(df, service, az_config):
logging.info(f'Starting write to storage table {az_config.OUTPUT_TABLE}.')
batch_counter = 0
batch = TableBatch()
for idx, row in df.iterrows():
task = {
'PartitionKey': row['PartitionKey'],
'RowKey': str(row['RowKey']),
'json_data': EntityProperty(EdmType.STRING, row['json_data'])
}
batch_counter += 1
batch.insert_entity(task)
if batch_counter == 100:
batch_counter = 0
service.commit_batch(az_config.OUTPUT_TABLE, batch)
batch = None
batch = TableBatch()
if batch_counter > 0:
service.commit_batch(az_config.OUTPUT_TABLE, batch)
logging.info('Finished write to storage table.')
def update_table(currency, day, rbf, linear, poly, actual):
print("Updating table: ", "\nCurrency: ", currency, "\nDay: ", day,
"\nRBF: ", rbf, "\nLinear: ", linear, "\nPolynomial: ", poly,
"\nActual: ", actual)
table_service = TableService(
connection_string=tablestorageconnectionstring)
entity = {
'PartitionKey': currency,
'RowKey': day,
'RBF': EntityProperty(EdmType.DOUBLE, rbf),
'Linear': EntityProperty(EdmType.DOUBLE, linear),
'Polynomial': EntityProperty(EdmType.DOUBLE, poly),
'Actual': EntityProperty(EdmType.DOUBLE, actual)
}
table_service.insert_or_replace_entity('PredictionData', entity)
def post(self, mail):
'''
Adds a message
'''
if request.is_json:
try:
try:
table_service.get_entity('users', mail, '')
message = Entity()
details = message_pb2.Message()
details.title = request.json["title"]
details.content = request.json["content"]
details.magic_number = request.json["magic_number"]
message.PartitionKey = mail
message.RowKey = str(
len(list(table_service.query_entities('messages'))) +
1)
message.details = EntityProperty(EdmType.BINARY,
MessageToJson(details))
table_service.insert_entity('messages', message)
return None, 201
except (Azure404):
return None, 404
except (KeyError):
return 'Please provide a json object conforming to the \
following pattern: {\"title\": \"Message title\", \
\"content\":\"Message content\", \
\"magic_number\": a number}', 400
def add_entity(currency, day, rbf, linear, poly, actual, comboV1):
print("Added Entity: ", "\nCurrency: ", currency, "\nDay: ", day,
"\nRBF: ", rbf, "\nLinear: ", linear, "\nPolynomial: ", poly,
"\nActual: ", actual, "\nComboV1: ", comboV1)
entity = {
'PartitionKey': currency,
'RowKey': day,
'RBF': EntityProperty(EdmType.DOUBLE, rbf),
'Linear': EntityProperty(EdmType.DOUBLE, linear),
'Polynomial': EntityProperty(EdmType.DOUBLE, poly),
'Actual': EntityProperty(EdmType.DOUBLE, actual),
'ComboV1': EntityProperty(EdmType.DOUBLE, comboV1)
}
entities.append(entity)
if (len(entities) > 99):
uploadentities()
def get_next_region_name():
# Number of regions available for deployment (in the table).
num_regions = table_service.get_entity(
table_name, part_table_control, "NumberOfRegions"
).NumberOfRegions.value
logging.debug("num_regions: {}".format(num_regions))
# Rowkey for the last region used (from the table).
last_region_rowkey_used = get_last_region_rowkey_used()
logging.debug("last_region_rowkey_used: {}".format(last_region_rowkey_used))
# Use the next region in the table. If the last region used was the last
# region in the table, then this time use the first region in the table.
curr_region_rowkey = last_region_rowkey_used + 1
if curr_region_rowkey > num_regions:
curr_region_rowkey = 1
logging.debug("curr_region_rowkey: {}".format(curr_region_rowkey))
# Cooldown. Select regions whose last successful run was > 48 hours ago
# or haven't had a last successful run.
orig_rowkey = curr_region_rowkey
last_rowkey = curr_region_rowkey - 1
if last_rowkey < 1:
last_rowkey = num_regions
logging.debug("last_rowkey = {}".format(last_rowkey))
while (hours_since_last_success(curr_region_rowkey) < cooldown_hours and
curr_region_rowkey != last_rowkey):
logging.debug("INFO: {0:.2f} hours since last successful run in region {1}. Skipping.".format(
hours_since_last_success(curr_region_rowkey),
get_region_shortname(curr_region_rowkey)
))
curr_region_rowkey += 1
if curr_region_rowkey > num_regions:
curr_region_rowkey = 1
if curr_region_rowkey == last_rowkey:
curr_region_rowkey = orig_rowkey
logging.debug("curr_region_rowkey (after cooldown): {}".format(
curr_region_rowkey))
# Get the region shortname.
region = get_region_shortname(curr_region_rowkey)
logging.debug("region: {}".format(region))
# Update LastRegionRowKeyUsed.
table_service.update_entity(
table_name, {
"PartitionKey": part_table_control,
"RowKey": "LastRegionRowKeyUsed",
"LastRegionRowKeyUsed":
EntityProperty(EdmType.INT32, curr_region_rowkey)
})
return region
def azureTable():
# table_service = TableService(account_name='', account_key='')
table_service = TableService(
connection_string=
'DefaultEndpointsProtocol=https;AccountName=sauokgp;AccountKey=113mdwUqIiqt4K2HonK80HakIOplxYZINmQME5KB1IZfP+v3JHZK64wpoTP5NBFaG0MaO/TVqA0nW4KuCINTow==;EndpointSuffix=core.windows.net'
)
#table_service.create_table('bitcoinData')
i = 0
for t in time:
bitcoin = {
'PartitionKey': Bitcoinz,
'RowKey': str(t),
'Open': EntityProperty(EdmType.DOUBLE, openz[i]),
'Close': EntityProperty(EdmType.DOUBLE, close[i]),
'High': EntityProperty(EdmType.DOUBLE, high[i]),
'Low': EntityProperty(EdmType.DOUBLE, low[i]),
'Time': timez[i],
'priority': 200
}
table_service.insert_entity('bitcoinData', bitcoin)
i += 1
def post(self):
"""
Inserts user into an Azure table.
"""
if request.is_json:
try:
user = Entity()
user.PartitionKey = request.json["email"]
user.RowKey = ''
user.info = EntityProperty(EdmType.BINARY, dumps(request.json))
table_service.insert_or_replace_entity('users', user)
except (KeyError):
return 'Please provide a json object conforming to \
the following pattern: {\"email\":\"[email protected]\",\
\"password\":\"xxx\", \"full_name\":\"Foo Bar\"}', 400
return None, 201
else:
return 'Please supply a json object in order to add a user.', 400
def _from_entity_int32(value):
return EntityProperty(EdmType.INT32, int(value))
def _from_entity_binary(value):
return EntityProperty(EdmType.BINARY, _decode_base64_to_bytes(value))
def _convert_json_to_entity(entry_element, property_resolver,
encrypted_properties):
''' Convert json response to entity.
The entity format is:
{
"Address":"Mountain View",
"Age":23,
"AmountDue":200.23,
"*****@*****.**":"Edm.Guid",
"CustomerCode":"c9da6455-213d-42c9-9a79-3e9149a57833",
"*****@*****.**":"Edm.DateTime",
"CustomerSince":"2008-07-10T00:00:00",
"IsActive":true,
"*****@*****.**":"Edm.Int64",
"NumberOfOrders":"255",
"PartitionKey":"mypartitionkey",
"RowKey":"myrowkey"
}
'''
entity = Entity()
properties = {}
edmtypes = {}
odata = {}
for name, value in entry_element.items():
if name.startswith('odata.'):
odata[name[6:]] = value
elif name.endswith('@odata.type'):
edmtypes[name[:-11]] = value
else:
properties[name] = value
# Partition key is a known property
partition_key = properties.pop('PartitionKey', None)
if partition_key:
entity['PartitionKey'] = partition_key
# Row key is a known property
row_key = properties.pop('RowKey', None)
if row_key:
entity['RowKey'] = row_key
# Timestamp is a known property
timestamp = properties.pop('Timestamp', None)
if timestamp:
entity['Timestamp'] = _from_entity_datetime(timestamp)
for name, value in properties.items():
mtype = edmtypes.get(name)
# use the property resolver if present
if property_resolver:
# Clients are not expected to resolve these interal fields.
# This check avoids unexpected behavior from the user-defined
# property resolver.
if not (name == '_ClientEncryptionMetadata1'
or name == '_ClientEncryptionMetadata2'):
mtype = property_resolver(partition_key, row_key, name, value,
mtype)
# throw if the type returned is not a valid edm type
if mtype and mtype not in _EDM_TYPES:
raise AzureException(
_ERROR_TYPE_NOT_SUPPORTED.format(mtype))
# If the property was encrypted, supercede the results of the resolver and set as binary
if encrypted_properties is not None and name in encrypted_properties:
mtype = EdmType.BINARY
# Add type for Int32
if type(value) is int:
mtype = EdmType.INT32
# no type info, property should parse automatically
if not mtype:
entity[name] = value
else: # need an object to hold the property
conv = _ENTITY_TO_PYTHON_CONVERSIONS.get(mtype)
if conv is not None:
try:
property = conv(value)
except Exception as e:
# throw if the type returned by the property resolver
# cannot be used in the conversion
if property_resolver:
raise AzureException(
_ERROR_INVALID_PROPERTY_RESOLVER.format(
name, value, mtype))
else:
raise e
else:
property = EntityProperty(mtype, value)
entity[name] = property
# extract etag from entry
etag = odata.get('etag')
if timestamp:
etag = 'W/"datetime\'' + url_quote(timestamp) + '\'"'
entity['etag'] = etag
return entity
def _encrypt_entity(entity, key_encryption_key, encryption_resolver):
'''
Encrypts the given entity using AES256 in CBC mode with 128 bit padding.
Will generate a content-encryption-key (cek) to encrypt the properties either
stored in an EntityProperty with the 'encrypt' flag set or those
specified by the encryption resolver. This cek is then wrapped using the
provided key_encryption_key (kek). Only strings may be encrypted and the
result is stored as binary on the service.
:param entity:
The entity to insert. Could be a dict or an entity object.
:param object key_encryption_key:
The user-provided key-encryption-key. Must implement the following methods:
wrap_key(key)--wraps the specified key using an algorithm of the user's choice.
get_key_wrap_algorithm()--returns the algorithm used to wrap the specified symmetric key.
get_kid()--returns a string key id for this key-encryption-key.
:param function(partition_key, row_key, property_name) encryption_resolver:
A function that takes in an entities partition key, row key, and property name and returns
a boolean that indicates whether that property should be encrypted.
:return: An entity with both the appropriate properties encrypted and the
encryption data.
:rtype: object
'''
_validate_not_none('entity', entity)
_validate_not_none('key_encryption_key', key_encryption_key)
_validate_key_encryption_key_wrap(key_encryption_key)
# AES256 uses 256 bit (32 byte) keys and always with 16 byte blocks
content_encryption_key = os.urandom(32)
entity_initialization_vector = os.urandom(16)
encrypted_properties = []
encrypted_entity = Entity()
for key, value in entity.items():
# If the property resolver says it should be encrypted
# or it is an EntityProperty with the 'encrypt' property set.
if (isinstance(value, EntityProperty) and value.encrypt) or \
(encryption_resolver is not None \
and encryption_resolver(entity['PartitionKey'], entity['RowKey'], key)):
# Only strings can be encrypted and None is not an instance of str.
if isinstance(value, EntityProperty):
if value.type == EdmType.STRING:
value = value.value
else:
raise ValueError(_ERROR_UNSUPPORTED_TYPE_FOR_ENCRYPTION)
if not isinstance(value, str):
raise ValueError(_ERROR_UNSUPPORTED_TYPE_FOR_ENCRYPTION)
# Value is now confirmed to hold a valid string value to be encrypted
# and should be added to the list of encrypted properties.
encrypted_properties.append(key)
propertyIV = _generate_property_iv(entity_initialization_vector,
entity['PartitionKey'],
entity['RowKey'], key, False)
# Encode the strings for encryption.
value = value.encode('utf-8')
cipher = _generate_AES_CBC_cipher(content_encryption_key,
propertyIV)
# PKCS7 with 16 byte blocks ensures compatibility with AES.
padder = PKCS7(128).padder()
padded_data = padder.update(value) + padder.finalize()
# Encrypt the data.
encryptor = cipher.encryptor()
encrypted_data = encryptor.update(
padded_data) + encryptor.finalize()
# Set the new value of this key to be a binary EntityProperty for proper serialization.
value = EntityProperty(EdmType.BINARY, encrypted_data)
encrypted_entity[key] = value
encrypted_properties = dumps(encrypted_properties)
# Generate the metadata iv.
metadataIV = _generate_property_iv(entity_initialization_vector,
entity['PartitionKey'],
entity['RowKey'],
'_ClientEncryptionMetadata2', False)
encrypted_properties = encrypted_properties.encode('utf-8')
cipher = _generate_AES_CBC_cipher(content_encryption_key, metadataIV)
padder = PKCS7(128).padder()
padded_data = padder.update(encrypted_properties) + padder.finalize()
encryptor = cipher.encryptor()
encrypted_data = encryptor.update(padded_data) + encryptor.finalize()
encrypted_entity['_ClientEncryptionMetadata2'] = EntityProperty(
EdmType.BINARY, encrypted_data)
encryption_data = _generate_encryption_data_dict(
key_encryption_key, content_encryption_key,
entity_initialization_vector)
encrypted_entity['_ClientEncryptionMetadata1'] = dumps(encryption_data)
return encrypted_entity
def update_available_region_names():
# Authenticate to Azure Subscription Manager
sub_client = SubscriptionClient(
ServicePrincipalCredentials(client_id=os.environ["AZURE_CLIENT_ID"],
secret=os.environ["AZURE_CLIENT_SECRET"],
tenant=os.environ["AZURE_TENANT_ID"]))
# Get the list of locations accessible to this subscription.
sub_id = os.environ["AZURE_SUBSCRIPTION_ID"]
locations = sorted(
[x.name for x in sub_client.subscriptions.list_locations(sub_id)] +
regions_to_include)
locations = [x for x in locations if x not in regions_to_exclude]
# Get entities for all of the regions.
old_entities = table_service.query_entities(
table_name, filter="PartitionKey eq '{}'".format(part_deploy_region))
old_entities_dict = {x.RegionShortname: x for x in old_entities}
# Create a new list of entities for the table.
new_entities = []
for i, loc in enumerate(locations):
# Remove this region from the original list and table.
old_entity = old_entities_dict.pop(loc, {})
# Save the last successful run timestamp (if it exists).
last_succ_run = None
if "LastSuccessfulRun" in old_entity:
last_succ_run = old_entity.LastSuccessfulRun
# Create a new entity for this region (re-indexed).
new_entities.append(
Entity(PartitionKey=part_deploy_region,
RowKey=str(i + 1),
RegionShortname=loc,
LastSuccessfulRun=last_succ_run))
logging.debug("new_entities = {}".format(new_entities))
# If there are any entries left in old_entities_dict, then that means
# some regions that were live before are no longer live.
if len(old_entities_dict):
logging.debug(
"WARNING: The following regions seem to have been decomissioned:")
for k, v in old_entities_dict.items():
logging.debug("\t{}".format(k))
# Delete the old entities.
for oe in old_entities:
table_service.delete_entity(table_name, oe.PartitionKey, oe.RowKey)
# Add the new entities.
for ne in new_entities:
table_service.insert_entity(table_name, ne)
# Update the last region RowKey (the number of new entities).
last_rowkey_entity = table_service.get_entity(table_name,
part_table_control,
"NumberOfRegions")
last_rowkey_entity.pop("etag", None)
last_rowkey_entity.pop("Timestamp", None)
last_rowkey_entity.VALUE = EntityProperty(EdmType.INT32, len(new_entities))
logging.debug("last_rowkey_entity = {}".format(last_rowkey_entity))
table_service.update_entity(table_name, last_rowkey_entity)
'device_id=e4ba4977314e200234880bac6b3585af; s=f61bvmhzj7; _ga=GA1.2.1562254249.1522590556; _gid=GA1.2.447715148.1528814975; xq_a_token=019174f18bf425d22c8e965e48243d9fcfbd2cc0; xq_a_token.sig=_pB0kKy3fV9fvtvkOzxduQTrp7E; xq_r_token=2d465aa5d312fbe8d88b4e7de81e1e915de7989a; xq_r_token.sig=lOCElS5ycgbih9P-Ny3cohQ-FSA; Hm_lvt_1db88642e346389874251b5a1eded6e3=1528814975,1528897343; Hm_lpvt_1db88642e346389874251b5a1eded6e3=1528897343; u=111528897342585; _gat_gtag_UA_16079156_4=1'
}
req = urllib.request.Request(url, headers=headers)
response = urllib.request.urlopen(req)
jdata = json.loads(response.read())
if self.total_symbols_to_crawl == 0:
self.total_symbols_to_crawl = int(jdata['count']['count'])
for eachStock in jdata['stocks']:
self.stocks.append(
Stock(eachStock['symbol'], eachStock['code'],
eachStock['name']))
self.total_symbols_to_crawled += 1
print('current parsed to page {}, {} symbols queried'.format(
self.current_page, self.total_symbols_to_crawled))
if __name__ == '__main__':
crawler = SymbolCrawler()
while crawler.has_next():
crawler.crawl_next()
table_service = TableService(account_name='heaventextb06a',
account_key='**fill your own key**')
for each_stock in crawler.stocks:
print(each_stock)
body = EntityProperty(EdmType.STRING, str(each_stock))
table_service.insert_or_replace_entity('heavenstock', each_stock)
