def validate_user_in_template(userId, templateId): table_service = TableService( connection_string=os.environ['AZURE_STORAGE_CONNECTION_STRING']) try: table_service.get_entity('user', templateId, userId) except: abort(403, description="user_id not found in template_id")
def validate_submission_in_template(): templateId = request.form['template_id'] submissionId = request.form['submission_id'] table_service = TableService( connection_string=os.environ['AZURE_STORAGE_CONNECTION_STRING']) try: table_service.get_entity('submission', templateId, submissionId) except: abort(400, description="submission_id not found in template_id")
def throttle_ip_requests(ip_entry): max_from_single_ip = 5 table_service = TableService(account_name=os.environ['STORAGE_ACCOUNT'], account_key=os.environ['STORAGE_KEY']) table_service.create_table( table_name=os.environ['BILLING_TABLE']) #create if it doesn't exist ip_row = None try: ip_row = table_service.get_entity(os.environ['BILLING_TABLE'], ip_entry['PartitionKey'], ip_entry['RowKey']) except: pass if not ip_row: ip_entry['count'] = 1 table_service.insert_entity(table_name=os.environ['BILLING_TABLE'], entity=ip_entry) ip_row = ip_entry else: lastdatetime = datetime.strptime(ip_row['time'], "%d/%m/%Y %H:%M:%S") currdatetime = datetime.strptime(ip_entry['time'], "%d/%m/%Y %H:%M:%S") tdelta = currdatetime - lastdatetime if tdelta.days < 1 and ip_row['count'] > max_from_single_ip: return True # throttle this entry.. elif tdelta.days > 0: #over 1 day has passed, update the count to 1 and reset time ip_row['count'] = 1 ip_row['time'] = currdatetime.strftime("%d/%m/%Y %H:%M:%S") table_service.update_entity(os.environ['BILLING_TABLE'], ip_row) else: # less than 1 day but count is < max_from_single_ip, update the count ip_row['count'] = ip_row['count'] + 1 table_service.update_entity(os.environ['BILLING_TABLE'], ip_row) # However we got here, do not throttle return False
def main(req: func.HttpRequest) -> func.HttpResponse: logging.info("Python HTTP trigger function processed a request.") path = req.route_params.get("path") if path is None: return func.HttpResponse(body=f"The url requested does not exist!", status_code=400) else: # connect to table storage_account_name = os.environ["STORAGE_ACCOUNT_NAME"] storage_account_key = os.environ["STORAGE_ACCOUNT_KEY"] table_service = TableService(account_name=storage_account_name, account_key=storage_account_key) # get longurl try: row = table_service.get_entity("links", "links", path) except: return func.HttpResponse( body=f"The url requested - {path} - does not exist!", status_code=400) else: longurl = row["LongUrl"] return func.HttpResponse(headers={"location": longurl}, status_code=302)
class AzureCosmosDb(TableStorage): def __init__(self, config: AzureCosmosDbConfig): self._tableService = TableService(account_name=config.account_name, account_key=config.account_key) self._tableName = config.table_name def check_entry_exists(self, entry): try: self.query(entry['PartitionKey'], entry['RowKey']) return True except: return False def write(self, entry): prepared = entry_storage.EntryOperations.prepare_entry_for_insert( entry) if not self.check_entry_exists(prepared): self._tableService.insert_entity(self._tableName, prepared) else: self._tableService.update_entity(self._tableName, prepared) def query(self, partitionkey, rowkey): task = self._tableService.get_entity(self._tableName, partitionkey, rowkey) return task def delete(self, partitionkey, rowkey): self._tableService.delete_entity(self._tableName, partitionkey, rowkey)
class AzureTable(): def __init__(self, account_name, account_key): self.table_service = TableService(account_name=account_name, account_key=account_key) def create_table(self, table_name): return self.table_service.create_table(table_name) def exists_table(self, table_name): return self.table_service.exists(table_name) def insert_or_replace_entity(self, table_name, partition_key, row_key, **kwargs): try: entity = self.table_service.get_entity(table_name, partition_key, row_key) except Exception: # Insert a new entity entity = {'PartitionKey': partition_key, 'RowKey': row_key} for (k, v) in kwargs.items(): entity[k] = v return self.table_service.insert_or_replace_entity(table_name, entity) def insert_or_replace_entity2(self, table_name, entity): return self.table_service.insert_or_replace_entity(table_name, entity) def insert_entity(self, table_name, entity): return self.table_service.insert_entity(table_name, entity) def update_entity(self, table_name, entity): return self.table_service.update_entity(table_name, entity) def get_entity(self, table_name, partition_key, row_key): return self.table_service.get_entity(table_name, partition_key, row_key) def delete_entity(self, table_name, partition_key, row_key): self.table_service.delete_entity(table_name, partition_key, row_key) def delete_table(self, table_name): return self.table_service.delete_table(table_name) def get_entities(self, table_name, partition_key): filter = "PartitionKey eq '{0}'".format(partition_key) return self.table_service.query_entities(table_name, filter)
def get_widget_telemetry(tbl_cnxn : TableService, widget : Widget): """ takes a widget and appends the telemetry to it. """ partition_key = str(widget.factory_id) + ":" + str(widget.line_id) row_key = str(widget.factory_id) + "-" + str(widget.serial_number) telemetry = tbl_cnxn.get_entity("Predictions", partition_key, row_key) return telemetry
def main(req: func.HttpRequest, ipListRowIn, ipListRowOut) -> func.HttpResponse: logging.info('Python HTTP trigger function processed a request.') # read request body reqMessage = req.get_json() logging.debug(reqMessage) # if available, load previous ip addressess from storage previousIpAddresses = None dataRow = json.loads(ipListRowIn) if len(dataRow) > 0: previousIpAddresses = json.loads(dataRow[0]['IpAddresses']) # read env vars tenantId = os.getenv("TENANT_ID") clientId = os.getenv("CLIENT_ID") clientSecret = os.getenv("CLIENT_SECRET") tableName = os.getenv("TABLE_NAME") entityRowKey = os.getenv("ENTITY_ROW_KEY") storageTableConnString = os.getenv("STORAGE_TABLE_CONN_STRING") # obtain access token address from Azure AD adservice = AzureAdService.AzureAdService(logging) accessToken = adservice.getAccessToken(tenantId, clientId, clientSecret) logging.debug('Azure AD access token: {0}'.format(accessToken)) # obtain current possible IP addresses from web/func app appservice = WebAppService.WebAppService(logging) possibleOutboundIps = appservice.getPossibleIpAddresses(accessToken, reqMessage['webAppSubscriptionId'], reqMessage['webAppResourceGroup'], reqMessage['webAppName']) # convert to json possibleOutboundIps = possibleOutboundIps.split(',') logging.debug('ip addresses: {0}'.format(possibleOutboundIps)) analysisservice = AzureAnalysisService.AzureAnalysisService(logging) analysisservice.updateFirewallSettings( accessToken, previousIpAddresses, possibleOutboundIps, AnalysisServerModel.AnalysisServerModel(reqMessage['analysisServerSubscriptionId'], reqMessage['analysisServerResourceGroup'], reqMessage['analysisServerName'])) # upsert table row if previousIpAddresses is None: data = { "PartitionKey": entityRowKey, "RowKey": entityRowKey, "IpAddresses": possibleOutboundIps } # insert using table storage output binding ipListRowOut.set(json.dumps(data)) else: # update existing row using table storage client library table_service = TableService(connection_string=storageTableConnString) table_row = table_service.get_entity(tableName, entityRowKey, entityRowKey) table_row.IpAddresses = json.dumps(possibleOutboundIps) table_service.update_entity(tableName, table_row) return func.HttpResponse('done', status_code=200)
def tableStorage(table_name, partition_key, row_key, hins_processed, timesaved, time_by_system, time_by_user, requests): try: table_service = TableService( account_name=config.AZURE['STORAGE_ACCOUNT_NAME'], account_key=config.AZURE['STORAGE_ACCOUNT_KEY']) entity = { 'PartitionKey': partition_key, 'RowKey': row_key, 'HinsProcessed': hins_processed, 'TimeSaved': timesaved, 'TimeBySystem': time_by_system, 'TimeByUser': time_by_user, 'Requests': requests } if not table_service.exists(table_name, timeout=None): table_service.create_table(table_name, fail_on_exist=False) try: table_service.insert_entity(table_name, entity) print("Entity Doesn't Exist") print("Creating Entity\n") except Exception as e: print("Entity Exists") print("Updating entity\n") currentEntity = table_service.get_entity(table_name, partition_key, row_key) tempHinProcessed = currentEntity.HinsProcessed + hins_processed tempTimeSaved = currentEntity.TimeSaved + timesaved tempTimeBySystem = currentEntity.TimeBySystem + time_by_system tempTimeByUser = currentEntity.TimeByUser + time_by_user tempRequest = currentEntity.Requests + requests entity = { 'PartitionKey': partition_key, 'RowKey': row_key, 'HinsProcessed': tempHinProcessed, 'TimeSaved': tempTimeSaved, 'TimeBySystem': tempTimeBySystem, 'TimeByUser': tempTimeByUser, 'Requests': tempRequest } table_service.update_entity(table_name, entity, if_match='*', timeout=None) except Exception as e: print(e)
def get_object(row_key, table_name='default', account_name='cloudmaticafunc9b4c', account_key='YOUR_KEY', partition_key='default'): table_service = TableService(account_name=account_name, account_key=account_key) obj = table_service.get_entity(table_name=table_name, partition_key=partition_key, row_key=row_key) return obj
def userpage(variable): table_service = TableService(account_name=app.config['StorageName'], account_key=app.config['StorageKey']) name = variable.lower() try: details = table_service.get_entity('weddingtable', 'Invites', name) print(details) return render_template("user.html", People1=details.Names, People2=details.Names2, hide=details.Hide, userCode=variable, commentmessage=details.Message) except: return redirect('/badCode')
def get_result(partition_key, row_key, tbl_svc_retry=None, **kwargs): """ Retirieves a result from the ``rosiepi`` storage table. :param: partition_key: The ``PartitionKey`` of the entity :param: row_key: The ``RowKey`` of the entity :param: **kwargs: Any additional kwargs to pass onto the ``TableService.get_entity()`` function. :return: The ``TableService.Entity``, or None if the Entity isn't found. """ response = None table = TableService(connection_string=os.environ['APP_STORAGE_CONN_STR']) if tbl_svc_retry is not None: table.retry = tbl_svc_retry # ensure RowKey is properly padded padding = '0'*(50 - len(row_key)) row_key = f'{padding}{row_key}' try: response = table.get_entity('rosiepi', partition_key, row_key, **kwargs) except Exception as err: logging.info(f'Failed to get result from rosiepi table. Error: {err}') raise logging.info(f'TableService.Entity retrieved: {response}') # Flatten the entity json_data_copy = response.get('json_data') if json_data_copy: del response['json_data'] try: flat_data = json.loads(json_data_copy) response.update(flat_data) except Exception as err: logging.info('Failed to flatten TableService.Entity.') raise for item in IGNORED_ITEMS: if item in response: del response[item] return response
def main(req: func.HttpRequest) -> func.HttpResponse: logging.info('Python HTTP Submit trigger received a request') logging.debug('Creating blob service') headers_dict = { "Access-Control-Allow-Credentials": "true", "Access-Control-Allow-Origin": "*", "Access-Control-Allow-Methods": "Post" } schema = put_schema.DoneSchema() try: done_dict = schema.loads(req.get_body()) except ValidationError: error = f'Failed to validate the done message' return func.HttpResponse(error, headers=headers_dict, status_code=400 ) table_name = os.getenv('AZURE_TABLE_NAME') table_service = TableService( account_name=os.getenv('AZURE_STORAGE_ACCOUNT'), account_key=os.getenv('AZURE_STORAGE_ACCESS_KEY') ) try: entity = table_service.get_entity(table_name, 'processing', done_dict['guid']) except AzureMissingResourceHttpError: error = f'Failed to put done message' return func.HttpResponse(error, headers=headers_dict, status_code=400 ) if not done_dict['error']: entity.Error = "" else: entity.Error = done_dict['error'] table_service.delete_entity(table_name, entity.PartitionKey, entity.RowKey) entity.PartitionKey = 'done' table_service.insert_entity(table_name, entity) return func.HttpResponse('Message was successfully inserted into Done queue', headers=headers_dict )
class State: def __init__(self): with open('./vardhamanbot/bot/config/app.json') as app_config_file: app_config = json.load(app_config_file) self.tableservice = TableService(app_config["ats_name"], app_config["ats_key"]) self.tableservice.retry = ExponentialRetry(initial_backoff=30, increment_base=2, max_attempts=20).retry def getStudent(self, userId): try: #Try to get the user data based on his year return self.tableservice.get_entity('vbotusers', "VMEG", userId) except azure.common.AzureMissingResourceHttpError: return {} def insertOrUpdateStudent(self, studentdata): #Partition the Data Based on Year studentdata["PartitionKey"] = "VMEG" self.tableservice.insert_or_replace_entity("vbotusers", studentdata)
def test_update(self): release = make_mag_release(self.account_name, self.account_key, 2019, 6, 1) try: release.create() # Update release release.state = MagState.archived release.archived_date = pendulum.utcnow().microsecond_(0) release.update() # Verify that release is updated service = TableService(account_name=self.account_name, account_key=self.account_key) entity = service.get_entity(MagRelease.TABLE_NAME, release.partition_key, release.row_key) updated_release = MagRelease.from_entity(entity) self.assertEqual(release.state, updated_release.state) self.assertEqual(release.archived_date, updated_release.archived_date) finally: release.delete()
def table(): account_name = config.STORAGE_ACCOUNT_NAME account_key = config.STORAGE_ACCOUNT_KEY table_service = TableService(account_name=account_name, account_key=account_key) table_name = config.TABLE_NAME #table_service.create_table(table_name) imageId = str(uuid.uuid4()) task = Entity() task.PartitionKey = 'dlws' task.RowKey = imageId task.description = 'test' table_service.insert_or_replace_entity(table_name, task) task = table_service.get_entity(table_name, 'dlws', imageId) print(task.description) tasks = table_service.query_entities('tasktable') for task in tasks: print(task.description) print(task.RowKey)
def getFaceImage(table: TableService, rowkey, rect): escores_table = "escores" entity = table.get_entity(escores_table, partition_key, rowkey) fname = entity.ts blob_name = "hackgt19" blob_key = "24wGa1RHd0BnemSDBbqRzvvTAB7Qy4IAN28E9de6OLR98wxnFljJXnKaBtzqJd2F53SmtNZP2NnZCPZkeL6wlQ==" try: container_name = 'imageblobs' # Create the BlockBlockService that is used to call the Blob service for the storage account block_blob_service = BlockBlobService(account_name=blob_name, account_key=blob_key) blob_bytes = block_blob_service.get_blob_to_bytes(container_name, fname) logging.info("Downloaded '{}' Successful".format(fname)) img = Image.open(io.BytesIO(blob_bytes)) img = img.crop(rect['left'], rect['top'], rect['left'] + rect['width'], rect['top'] + rect['height']) return img except Exception as e: logging.error("Error: {}".format(e)) return None
def login(email, password): # Retrieve login information from our table table_service = TableService( account_name=azure_storage_credentials["account_name"], account_key=azure_storage_credentials["access_key"]) user = table_service.get_entity('user', '*****@*****.**', '') email = user["PartitionKey"] pepper = get_pepper() password = bytes(str(password + pepper).encode('utf-8')) password_hash = user["PasswordHash"].encode('utf-8') # Calculate a hash of our password + pepper #hashed = bcrypt.hashpw(password_pepper,bcrypt.gensalt()) #Login if bcrypt.checkpw(password, password_hash): print('Log in successful!') # TODO: generate and return JWT else: print('You have not logged in') # TODO: return error message asdf = "asdf"
class AzureOperationsStorage(BasicOperationStorage): """ Implementation of :class:`.interface.IOperationStorage` with Azure Table Storage using the default implementation :class:`.interface.BasicOperationStorage` On creating a connection to the storage is initialized and all needed tables are created. If a purge is necessary, tables are not deleted but simple the content removed. Table creation can take a while with Azure Table Storage. As Azure Table Storage only supports two indices, the operations are inserted multiple times in different tables to enable multi-index queries. """ def get_retry_exceptions(self): return (NewConnectionError) @retry_auto_reconnect def __init__(self, azure_config, purge=False): super(AzureOperationsStorage, self).__init__() if not azure_config: raise Exception("No azure table storage configuration provided!") self._azure_config = azure_config # ensure defaults self._azure_config["operation_table"] = self._azure_config.get( "operation_table", "operations") self._azure_config["address_table"] = self._azure_config.get( "address_table", "address") self._azure_config["status_table"] = self._azure_config.get( "status_table", "status") self._azure_config["balances_table"] = self._azure_config.get( "balances_table", "balances") if not self._azure_config["account"]: raise Exception( "Please include the azure account name in the config") if not self._azure_config["key"]: raise Exception( "Please include the azure account key in the config") self._service = TableService( account_name=self._azure_config["account"], account_key=self._azure_config["key"]) # if tables doesnt exist, create it self._create_operations_storage(purge) self._create_status_storage(purge) self._create_address_storage(purge) self._create_balances_storage(purge) def _debug_print(self, operation): from pprint import pprint pprint(operation) def _create_address_storage(self, purge): _varients = ["balance", "historyfrom", "historyto"] for variant in _varients: tablename = self._azure_config["address_table"] + variant if purge: try: for item in self._service.query_entities(tablename): self._service.delete_entity(tablename, item["PartitionKey"], item["RowKey"]) except AzureHttpError: pass except AzureMissingResourceHttpError: pass while not self._service.exists(tablename): self._service.create_table(tablename) time.sleep(0.1) def _create_status_storage(self, purge): if purge: try: tablename = self._azure_config["status_table"] for item in self._service.query_entities(tablename): self._service.delete_entity(tablename, item["PartitionKey"], item["RowKey"]) except AzureMissingResourceHttpError: pass while not self._service.exists(self._azure_config["status_table"]): self._service.create_table(self._azure_config["status_table"]) time.sleep(0.1) def _create_balances_storage(self, purge): if purge: try: tablename = self._azure_config["balances_table"] for item in self._service.query_entities(tablename): self._service.delete_entity(tablename, item["PartitionKey"], item["RowKey"]) except AzureMissingResourceHttpError: pass while not self._service.exists(self._azure_config["balances_table"]): self._service.create_table(self._azure_config["balances_table"]) time.sleep(0.1) def _create_operations_storage(self, purge): self._operation_varients = [ "incident", "statuscompleted", "statusfailed", "statusinprogress" ] # "customer" self._operation_tables = {} for variant in self._operation_varients: self._operation_tables[ variant] = self._azure_config["operation_table"] + variant self._operation_prep = { "statusinprogress": lambda op: { "PartitionKey": self._short_digit_hash(op["chain_identifier"]), "RowKey": op["chain_identifier"] }, "statuscompleted": lambda op: { "PartitionKey": self._short_digit_hash(op["chain_identifier"]), "RowKey": op["chain_identifier"] }, "statusfailed": lambda op: { "PartitionKey": self._short_digit_hash(op["chain_identifier"]), "RowKey": op["chain_identifier"] }, "customer": lambda op: { "PartitionKey": op["customer_id"], "RowKey": op["chain_identifier"] }, "incident": lambda op: { "PartitionKey": self._short_digit_hash(op["incident_id"]), "RowKey": op["incident_id"] } } for variant in self._operation_varients: if purge: try: tablename = self._operation_tables[variant] for item in self._service.query_entities(tablename): self._service.delete_entity(tablename, item["PartitionKey"], item["RowKey"]) except AzureMissingResourceHttpError: pass while not self._service.exists(self._operation_tables[variant]): self._service.create_table(self._operation_tables[variant]) time.sleep(0.1) def _get_with_ck(self, variant, operation): with_ck = operation.copy() with_ck.update(self._operation_prep[variant](with_ck)) return with_ck def _short_digit_hash(self, value): hash_type = Config.get("operation_storage", "key_hash", "type", default="crc32") if hash_type == "crc32": short_hash = hex(zlib.crc32(value.encode(encoding='UTF-8'))) short_hash = short_hash[2:len(short_hash)] elif hash_type == "sha256": checker = hashlib.sha256() checker.update(value.encode(encoding='UTF-8')) short_hash = checker.hexdigest() return short_hash[0:Config. get("operation_storage", "key_hash", "digits", 3)] @retry_auto_reconnect def track_address(self, address, usage="balance"): address = ensure_address_format(address) try: short_hash = self._short_digit_hash(address) logging.getLogger(__name__).debug("track_address with " + str(address) + ", hash " + str(short_hash)) self._service.insert_entity( self._azure_config["address_table"] + usage, { "PartitionKey": short_hash, "RowKey": address, "address": address, "usage": usage }) except AzureConflictHttpError: raise AddressAlreadyTrackedException @retry_auto_reconnect def untrack_address(self, address, usage="balance"): address = ensure_address_format(address) try: short_hash = self._short_digit_hash(address) logging.getLogger(__name__).debug("untrack_address with " + str(address) + ", hash " + str(short_hash)) self._service.delete_entity( self._azure_config["address_table"] + usage, short_hash, address) try: self._delete_balance(address) except AzureMissingResourceHttpError: pass except AzureMissingResourceHttpError: raise AddressNotTrackedException() @retry_auto_reconnect def _get_address(self, address, usage="balance"): try: short_hash = self._short_digit_hash(address) logging.getLogger(__name__).debug("_get_address with " + str(address) + ", hash " + str(short_hash)) return self._service.get_entity( self._azure_config["address_table"] + usage, short_hash, address) except AzureMissingResourceHttpError: raise AddressNotTrackedException() def _update(self, operation, status=None): try: mapping = { "in_progress": "statusinprogress", "completed": "statuscompleted", "failed": "statusfailed" } operation = self._get_with_ck("incident", operation.copy()) new_operation = operation if status: tmp = self.get_operation(operation["incident_id"]) new_operation["timestamp"] = tmp["timestamp"] new_operation["status"] = status new_operation = self._get_with_ck("incident", new_operation) logging.getLogger(__name__).debug( "_update: Table " + self._operation_tables["incident"] + " PartitionKey " + new_operation["PartitionKey"] + " " + new_operation["RowKey"]) self._service.update_entity(self._operation_tables["incident"], new_operation) operation = self._get_with_ck("statuscompleted", operation.copy()) new_operation = operation if status: tmp = self.get_operation(operation["incident_id"]) new_operation["timestamp"] = tmp["timestamp"] new_operation["status"] = status new_operation = self._get_with_ck("statuscompleted", new_operation) self._service.update_entity( self._operation_tables["statuscompleted"], new_operation) logging.getLogger(__name__).debug( "_update: Table " + self._operation_tables["statuscompleted"] + " PartitionKey " + new_operation["PartitionKey"] + " " + new_operation["RowKey"]) if status: # needs delete and insert try: self._service.delete_entity( self._operation_tables[mapping[operation["status"]]], operation["PartitionKey"], operation["RowKey"]) except AzureMissingResourceHttpError: pass try: self._service.insert_entity( self._operation_tables[mapping[ new_operation["status"]]], new_operation) except AzureConflictHttpError: # already exists, try update self._service.update_entity( self._operation_tables[mapping[ new_operation["status"]]], new_operation) else: self._service.update_entity( self._operation_tables[mapping[new_operation["status"]]], new_operation) except AzureMissingResourceHttpError: raise OperationNotFoundException() def _insert(self, operation): try: for variant in self._operation_varients: to_insert = operation.copy() to_insert.update(self._operation_prep[variant](to_insert)) if not to_insert["PartitionKey"]: raise AzureMissingResourceHttpError() if not to_insert["RowKey"]: raise AzureMissingResourceHttpError() logging.getLogger(__name__).debug( "_insert: Table " + self._operation_tables[variant] + " PartitionKey " + to_insert["PartitionKey"] + " " + to_insert["RowKey"]) self._service.insert_entity(self._operation_tables[variant], to_insert) except AzureConflictHttpError: raise DuplicateOperationException() def _delete(self, operation): try: for variant in self._operation_varients: to_delete = operation.copy() to_delete.update(self._operation_prep[variant](to_delete)) self._service.delete_entity(self._operation_tables[variant], to_delete["PartitionKey"], to_delete["RowKey"]) except AzureMissingResourceHttpError: raise OperationNotFoundException() @retry_auto_reconnect def flag_operation_completed(self, operation): # do basics operation = super(AzureOperationsStorage, self).flag_operation_completed(operation) self._update(operation, status="completed") self._ensure_balances(operation) @retry_auto_reconnect def flag_operation_failed(self, operation, message=None): # do basics operation = super(AzureOperationsStorage, self).flag_operation_failed(operation) operation["message"] = message self._update(operation, status="failed") @retry_auto_reconnect def insert_operation(self, operation): # do basics operation = super(AzureOperationsStorage, self).insert_operation(operation) error = None try: self._insert(operation) except DuplicateOperationException as e: error = e try: # always check if balances are ok if operation["status"] == "completed": self._ensure_balances(operation) except BalanceConcurrentException as e: if error is None: error = e if error is not None: raise error @retry_auto_reconnect def _delete_balance(self, address, if_match='*'): self._service.delete_entity(self._azure_config["balances_table"], self._short_digit_hash(address), address, if_match=if_match) @retry_auto_reconnect def _ensure_balances(self, operation): affected_address = get_tracking_address(operation) logging.getLogger(__name__).debug("_ensure_balances: with " + operation["chain_identifier"] + " for address " + str(affected_address)) try: self._get_address(affected_address) except AddressNotTrackedException: # delte if exists and return try: self._delete_balance(affected_address) except AzureMissingResourceHttpError: pass return try: balance_dict = self._service.get_entity( self._azure_config["balances_table"], self._short_digit_hash(affected_address), affected_address) insert = False except AzureMissingResourceHttpError as e: balance_dict = {"address": affected_address} balance_dict["PartitionKey"] = self._short_digit_hash( balance_dict["address"]) balance_dict["RowKey"] = balance_dict["address"] insert = True if operation["block_num"] < balance_dict.get("blocknum", 0): raise BalanceConcurrentException() elif operation["block_num"] == balance_dict.get("blocknum", 0) and\ operation["txnum"] < balance_dict.get("txnum", 0): raise BalanceConcurrentException() elif operation["block_num"] == balance_dict.get("blocknum", 0) and\ operation["txnum"] == balance_dict.get("txnum", 0) and\ operation["opnum"] <= balance_dict.get("opnum", 0): raise BalanceConcurrentException() balance_dict["blocknum"] = max(balance_dict.get("blocknum", 0), operation["block_num"]) balance_dict["txnum"] = max(balance_dict.get("txnum", 0), operation["tx_in_block"]) balance_dict["opnum"] = max(balance_dict.get("opnum", 0), operation["op_in_tx"]) total = 0 addrs = split_unique_address(affected_address) asset_id = "balance" + operation["amount_asset_id"].split("1.3.")[1] if addrs["account_id"] == operation["from"]: # internal transfer and withdraw # negative balance = balance_dict.get(asset_id, 0) balance_dict[asset_id] = balance - operation["amount_value"] # fee as well asset_id = operation["fee_asset_id"] balance = balance_dict.get(asset_id, 0) balance_dict[asset_id] = balance - operation["fee_value"] elif addrs["account_id"] == operation["to"]: # deposit # positive balance = balance_dict.get(asset_id, 0) balance_dict[asset_id] = balance + operation["amount_value"] # fees were paid by someone else else: raise InvalidOperationException() for key, value in balance_dict.items(): if key.startswith("balance"): total = total + value if total == 0: if not insert: try: self._delete_balance(affected_address, if_match=balance_dict.etag) except AzureMissingResourceHttpError: pass return # may be updated or inserted, total > 0 if (insert): try: self._service.insert_entity( self._azure_config["balances_table"], balance_dict) except AzureMissingResourceHttpError: raise OperationStorageException( "Critical error in database consistency") else: try: self._service.update_entity( self._azure_config["balances_table"], balance_dict, if_match=balance_dict.etag) except AzureConflictHttpError: raise OperationStorageException( "Critical error in database consistency") @retry_auto_reconnect def insert_or_update_operation(self, operation): # do basics operation = super(AzureOperationsStorage, self).insert_operation(operation) # check if this is from in_progress to complete (for withdrawals we need to find incident id as its # not stored onchain) try: logging.getLogger(__name__).debug( "insert_or_update_operation: check if in_progress with " + str(operation["chain_identifier"]) + " exists") existing_operation = self.get_operation_by_chain_identifier( "in_progress", operation["chain_identifier"]) logging.getLogger(__name__).debug( "insert_or_update_operation: found existing in_progress operation" ) if not existing_operation["incident_id"] == operation["incident_id"] and\ operation["incident_id"] == operation["chain_identifier"]: logging.getLogger(__name__).debug( "insert_or_update_operation: using preset incident_id " + str(existing_operation["incident_id"])) operation["incident_id"] = existing_operation["incident_id"] except OperationNotFoundException: existing_operation = None if existing_operation is None: try: logging.getLogger(__name__).debug( "insert_or_update_operation: attempting insert") error = None try: self._insert(operation) except DuplicateOperationException as e: error = e try: # always check if balances are ok if operation["status"] == "completed": self._ensure_balances(operation) except BalanceConcurrentException as e: if error is None: error = e if error is not None: raise error except DuplicateOperationException as ex: logging.getLogger(__name__).debug( "insert_or_update_operation: fallback to update") # could be an update to completed ... if operation.get("block_num"): try: operation.pop("status") self.flag_operation_completed(operation) except OperationNotFoundException: raise ex else: raise ex else: logging.getLogger(__name__).debug( "insert_or_update_operation: attempting update") if operation.get("block_num"): try: operation.pop("status") self.flag_operation_completed(operation) except OperationNotFoundException: raise ex @retry_auto_reconnect def delete_operation(self, operation_or_incident_id): # do basics operation = super(AzureOperationsStorage, self).delete_operation(operation_or_incident_id) if type(operation_or_incident_id) == str: operation = self.get_operation(operation_or_incident_id) else: operation = operation_or_incident_id self._delete(operation) @retry_auto_reconnect def get_operation_by_chain_identifier(self, status, chain_identifier): mapping = { "in_progress": "statusinprogress", "completed": "statuscompleted", "failed": "statusfailed" } try: operation = self._service.get_entity( self._operation_tables[mapping[status]], self._short_digit_hash(chain_identifier), chain_identifier) operation.pop("PartitionKey") operation.pop("RowKey") operation.pop("Timestamp") operation.pop("etag") except AzureMissingResourceHttpError: raise OperationNotFoundException() return operation @retry_auto_reconnect def get_operation(self, incident_id): try: short_hash = self._short_digit_hash(incident_id) logging.getLogger(__name__).debug("get_operation with " + str(incident_id) + ", hash " + str(short_hash)) operation = self._service.get_entity( self._operation_tables["incident"], short_hash, incident_id) operation.pop("PartitionKey") operation.pop("RowKey") operation.pop("Timestamp") operation.pop("etag") except AzureMissingResourceHttpError: raise OperationNotFoundException() return operation @retry_auto_reconnect def get_balances(self, take, continuation=None, addresses=None, recalculate=False): if recalculate: raise Exception( "Currently not supported due to memo change on withdraw") return self._get_balances_recalculate(take, continuation, addresses) else: if continuation is not None: try: continuation_marker = json.loads(continuation) except TypeError: raise InputInvalidException() except JSONDecodeError: raise InputInvalidException() balances = self._service.query_entities( self._azure_config["balances_table"], num_results=take, marker=continuation_marker) else: balances = self._service.query_entities( self._azure_config["balances_table"], num_results=take) return_balances = {} for address_balance in balances: return_balances[address_balance["address"]] = { "block_num": address_balance["blocknum"] } for key, value in address_balance.items(): if key.startswith("balance"): asset_id = "1.3." + key.split("balance")[1] return_balances[ address_balance["address"]][asset_id] = value return_balances["continuation"] = None if balances.next_marker: return_balances["continuation"] = json.dumps( balances.next_marker) return return_balances @retry_auto_reconnect def _get_balances_recalculate(self, take, continuation=None, addresses=None): address_balances = collections.defaultdict( lambda: collections.defaultdict()) if not addresses: if continuation is not None: try: continuation_marker = json.loads(continuation) except TypeError: raise InputInvalidException() except JSONDecodeError: raise InputInvalidException() addresses = self._service.query_entities( self._azure_config["address_table"] + "balance", num_results=take, marker=continuation_marker) else: addresses = self._service.query_entities( self._azure_config["address_table"] + "balance", num_results=take) if addresses.next_marker: address_balances["continuation"] = json.dumps( addresses.next_marker) addresses = [x["address"] for x in addresses] if type(addresses) == str: addresses = [addresses] for address in addresses: addrs = split_unique_address(address) max_block_number = 0 for operation in self.get_operations_completed( filter_by={"customer_id": addrs["customer_id"]}): this_block_num = operation["block_num"] asset_id = operation["amount_asset_id"] if addrs["account_id"] == operation["from"]: # negative balance = address_balances[address].get(asset_id, 0) address_balances[address][asset_id] =\ balance - operation["amount_value"] # fee as well asset_id = operation["fee_asset_id"] balance = address_balances[address].get(asset_id, 0) address_balances[address][asset_id] =\ balance - operation["fee_value"] elif addrs["account_id"] == operation["to"]: # positive balance = address_balances[address].get(asset_id, 0) address_balances[address][asset_id] =\ balance + operation["amount_value"] else: raise InvalidOperationException() max_block_number = max(max_block_number, this_block_num) if max_block_number > 0: address_balances[address]["block_num"] = max_block_number # do not return default dicts for key, value in address_balances.items(): if type(value) == collections.defaultdict: address_balances[key] = dict(value) return dict(address_balances) def _parse_filter(self, filter_by): if filter_by: if filter_by.get("customer_id"): return {"customer_id": filter_by.pop("customer_id")} if filter_by.get("address"): addrs = split_unique_address(filter_by.pop("address")) return {"customer_id": addrs["customer_id"]} if filter_by.get("from"): addrs = split_unique_address(filter_by.pop("from")) return {"from": addrs["account_id"]} if filter_by.get("to"): addrs = split_unique_address(filter_by.pop("to")) return {"to": addrs["account_id"]} if filter_by: raise Exception("Filter not supported") return {} def _filter_dict_to_string(self, filter_dict, partition_key=None): filter_str = None for key, value in filter_dict.items(): if partition_key == key: key = "PartitionKey" if filter_str is not None: delimiter = " and " delimiter = "" filter_str = delimiter + key + " eq '" + value + "'" return filter_str @retry_auto_reconnect def get_operations_in_progress(self, filter_by=None): mapping = { "in_progress": "statusinprogress", "completed": "statuscompleted", "failed": "statusfailed" } filter_dict = {} filter_dict.update(self._parse_filter(filter_by)) filter_str = self._filter_dict_to_string(filter_dict, "status") return list( self._service.query_entities( self._operation_tables[mapping["in_progress"]], filter_str)) @retry_auto_reconnect def get_operations_completed(self, filter_by=None): mapping = { "in_progress": "statusinprogress", "completed": "statuscompleted", "failed": "statusfailed" } filter_dict = {} filter_dict.update(self._parse_filter(filter_by)) filter_str = self._filter_dict_to_string(filter_dict, "status") return list( self._service.query_entities( self._operation_tables[mapping["completed"]], filter_str)) @retry_auto_reconnect def get_operations_failed(self, filter_by=None): mapping = { "in_progress": "statusinprogress", "completed": "statuscompleted", "failed": "statusfailed" } filter_dict = {} filter_dict.update(self._parse_filter(filter_by)) filter_str = self._filter_dict_to_string(filter_dict, "status") return list( self._service.query_entities( self._operation_tables[mapping["failed"]], filter_str)) @retry_auto_reconnect def get_last_head_block_num(self): try: document = self._service.get_entity( self._azure_config["status_table"], "head_block_num", "last") return document["last_head_block_num"] except AzureMissingResourceHttpError: return 0 @retry_auto_reconnect def set_last_head_block_num(self, head_block_num): current_last = self.get_last_head_block_num() if current_last >= head_block_num: raise Exception("Marching backwards not supported! Last: " + str(current_last) + " New: " + str(head_block_num)) self._service.insert_or_replace_entity( self._azure_config["status_table"], { "PartitionKey": "head_block_num", "RowKey": "last", "last_head_block_num": head_block_num })
class AzureTable(object): def __init__(self, account_name: str, account_key: str, table_name: str, partition_key_field: str, clustering_key_field: str): self.table = TableService(account_name=account_name, account_key=account_key) self.table_name = self.table_name self.partition_key_field = partition_key_field self.clustering_key_field = clustering_key_field @property def partition_key_name(self) -> str: return 'PartitionKey' @property def clustering_key_name(self) -> str: return 'RowKey' def get_payload(self, payload: dict): item = deepcopy(payload) partition_key = payload.get(self.partition_key_field) clustering_key = payload.get(self.clustering_key_field) if partition_key is None: raise PartitionKeyNotFoundError( 'payload={} does not have a partition key') if clustering_key is None: raise ClusteringKeyNotFoundError( 'payload={} does not have a clustering key') item.update({ self.partition_key_name: partition_key, self.clustering_key_name: clustering_key }) return item def create(self): return self.table.create_table(self.table_name) def insert(self, item: dict): return self.table.insert_entity(self.table_name, self.get_payload(item)) def update(self, item: dict): pass def upsert(self, item: dict): pass def delete(self, key: str): pass def read(self, key: str): pass def insert_batch(self, items: list): batch = TableBatch() for item in items: batch.insert_entity(self.get_payload(item)) return self.table.commit_batch(self.table_name, batch) def get(self, partition_key: str, clustering_key: str): return self.table.get_entity(self.table_name, partition_key, clustering_key) def get_by_partition(self, partition_key: str) -> list: return self.table.query_entities(self.table_name, filter="{} eq '{}'".format( self.partition_key_name, partition_key))
table_service.commit_batch('tasktable', batch) # alternative way to use batch, using context print("batch insert using context...") task006 = {'PartitionKey': 'tasksSeattle', 'RowKey': '006', 'description': 'Go grocery shopping', 'priority': 400} task007 = {'PartitionKey': 'tasksSeattle', 'RowKey': '007', 'newCol': 'newColVal1', 'description': 'Clean the bathroom', 'priority': 100} with table_service.batch('tasktable') as batch: batch.insert_entity(task006) batch.insert_entity(task007) # Query print("Query task 001...") task = table_service.get_entity('tasktable', 'tasksSeattle', '001') print(task.description) print(task.priority) # Query a set of entities print("Query set of entities...") tasks = table_service.query_entities( 'tasktable', filter="PartitionKey eq 'tasksSeattle'") for task in tasks: print(task.description) print(task.priority) try: print(task.newCol) except AttributeError: print("No newCol.")
with open(upload_file_path, "rb") as data: blob_client.upload_blob(data) table_service = TableService(account_name='<Name of storage account>', account_key='<TableStorage Account Key>') #table_service.create_table('CarDetails') #Insert Record car1 = {'PartitionKey': 'Car', 'RowKey' : '<NumberPlate>', 'OwnerName' : 'xxxxx', 'ContactNo' : '+91-xxxxx xxxxx', 'PollutionLevel' : 0, 'LastCheckedDate' : '2020-11-15', 'AmountToBePaid' : 0, 'TimeStamps' : ""} table_service.insert_entity('CarDetails', car1) #Query task = table_service.get_entity('CarDetails', 'Car', str(text)) mq = MQ(); perc = mq.MQPercentage() poll = perc["CO"] today = date.today() l = str(task.LastCheckedDate) l = date(int(l[0:4]), int(l[5:7]), int(l[8:])) delta = (today - l) if poll>500 and delta.days > 10: task.PollutionLevel = poll task.LastCheckedDate = today task.AmountToBePaid = task.AmountToBePaid + 100 task.TimeStamps = task.TimeStamps + "-" + str(ts) #Logic App Message = "Hello " + str(task.OwnerName) + ", It is observed that the pollution levels for your vehicle " + str(task.RowKey) + " is above the normal level. Please pay a fine of Rs. " + str(task.AmountToBePaid) + " online or in your nearest RTO office."
ser.write(bytearray([0x31])) time.sleep(delaytime) if __name__ == '__main__': # 時間標記 memoryTime = time.time() openDoor = False # 實體化 Azure Table table_service = TableService(account_name='acc_name', account_key='acc_key') print( table_service.get_entity('testAPPs', 'ComputerVision', 'instruct', select='action').action) # 無窮迴圈 while True: # 如果 Table 的 action 欄位是 open if table_service.get_entity('testAPPs', 'ComputerVision', 'instruct', select='action').action == 'open': # 開門 if openDoor == False: openDoor = True memoryTime = time.time() controlDoor('open') # 三秒後關門,並把 Table 的 action 欄位改為 close
class Azure_Storage(): def __init__(self, create_new=False): account_name = config.STORAGE_ACCOUNT_NAME account_key = config.STORAGE_ACCOUNT_KEY self.task_queue_name = config.TASK_QUEUE_NAME self.table_name = config.TABLE_NAME self.container_name = config.BLOB_CONTAINER_NAME self.ImagePartitionKey = config.IMAGE_PARTITION_KEY self.table_service = TableService(account_name=account_name, account_key=account_key) self.block_blob_service = BlockBlobService(account_name=account_name, account_key=account_key) self.queue_service = QueueService(account_name=account_name, account_key=account_key) if create_new: queue_service.create_queue(task_queue_name) block_blob_service.create_container(container_name) table_service.create_table(table_name) def put_image(self, image_uuid, image_bytes): ret = self.block_blob_service.create_blob_from_bytes( self.container_name, image_uuid, image_bytes) return ret def get_image(self, image_uuid): ret = self.block_blob_service.get_blob_to_bytes( self.container_name, image_uuid).content return ret def put_classification_result(self, image_uuid, results): task = Entity() task.PartitionKey = self.ImagePartitionKey task.RowKey = image_uuid task.results = str(results) ret = self.table_service.insert_or_replace_entity( self.table_name, task) return ret def get_classification_result(self, image_uuid): try: task = self.table_service.get_entity(self.table_name, self.ImagePartitionKey, image_uuid) return task.results except Exception as e: return None def put_task(self, taskmsg): ret = self.queue_service.put_message(self.task_queue_name, taskmsg) return ret #payload is in message.content def get_task(self, num_messages=16): messages = self.queue_service.get_messages(self.task_queue_name, num_messages=num_messages, visibility_timeout=1 * 60) return messages def delete_task(self, message): ret = self.queue_service.delete_message(self.task_queue_name, message.id, message.pop_receipt) return ret
'color': 'Green', 'price': '15' } table_service.insert_entity(table_name, entity_example) entity_example = { 'PartitionKey': 'Bike', 'RowKey': 'Audi', 'text': 'Germany', 'color': 'Green', 'price': '5' } table_service.insert_entity(table_name, entity_example) # COMMAND ---------- task = table_service.get_entity(table_name, 'Car', 'Tesla') print(task) # COMMAND ---------- tasks = table_service.query_entities(table_name, filter="price eq '5'") for task in tasks: print(task.PartitionKey) print(task.RowKey) print(task.text) print("---") # COMMAND ----------
class Storage_az_table(): ################################################################## # BEGIN - Common methods to implement storage interface def __init__(self): connection_string = os.environ['SHKOLA_AZ_TABLE_CONN_STR'] self.table_service = TableService(connection_string=connection_string) self.default_partition_key = "USER" self.users_table_name = 'users' self.responses_table_name = 'responses' self.sessions_table_name = 'sessions' self.feedbacks_table_name = 'feedbacks' tables = [ self.users_table_name, self.responses_table_name, self.sessions_table_name, self.feedbacks_table_name ] existing_tables = list( map(lambda x: x.name, self.table_service.list_tables())) for table in tables: if table not in existing_tables: self.table_service.create_table(table) def get_user(self, user_id): partition_key = self.default_partition_key try: entity = self.table_service.get_entity(self.users_table_name, partition_key, user_id) except AzureMissingResourceHttpError: return None entity["user_id"] = user_id return entity def update_user(self, user_id, name=None, remote_ip=None, user_agent=None, picture=None, user_language=None, selected_language=None, last_accessed=None): properties = dict() # Nothing better at the moment: properties['PartitionKey'] = self.default_partition_key properties['RowKey'] = user_id properties['user_id'] = user_id properties["name"] = name properties["remote_ip"] = remote_ip properties["user_agent"] = user_agent properties["picture"] = picture properties["user_language"] = user_language if selected_language: properties["selected_language"] = selected_language properties["last_accessed"] = last_accessed logging.debug("azure table update_user %s: %s", str(name), str(properties)) try: self.table_service.insert_or_merge_entity(self.users_table_name, properties) except Exception: logging.exception('Error adding to table ' + self.users_table_name + ' record: {}'.format(properties)) def update_selected_language(self, user_id, selected_language): properties = dict() # Nothing better at the moment: properties['PartitionKey'] = self.default_partition_key properties['RowKey'] = user_id properties['user_id'] = user_id properties["selected_language"] = selected_language logging.debug("azure table update_user_langauge %s: %s", str(user_id), str(properties)) try: self.table_service.insert_or_merge_entity(self.users_table_name, properties) except Exception: logging.exception('Error adding to table ' + self.users_table_name + ' record: {}'.format(properties)) def insert_user_id(self, user_id): self.update_user(user_id) return user_id # Get all user responses from the response table def get_all_user_results(self, u_id, from_date=None): req = "(PartitionKey eq '{}')".format(u_id) if from_date: if len(req) > 0: req = req + " and " req = req + "(Timestamp ge datetime'{}')".format(from_date) entries = self.table_service.query_entities(self.responses_table_name, req) result = [] for row in entries: result.append(row) return result # Update user info with the latest <stats>, and with # <stats_time> being the time of the latest stats def update_user_stats(self, user_id, stats, stats_time): properties = dict() # Nothing better at the moment: properties['PartitionKey'] = self.default_partition_key properties['RowKey'] = user_id properties['user_id'] = user_id properties['stats'] = encode_dict(stats) properties['stats_time'] = stats_time try: self.table_service.merge_entity(self.users_table_name, properties) except Exception: logging.exception('Error updating results for user ' + user_id + ' record: {}'.format(properties)) @timer_section("storage.record_response") def record_response(self, response): fb_time = int(time.time() * 1000) response['PartitionKey'] = response['user_id'] response['RowKey'] = response['question_id'] + "|" + \ str(response['attempt']) + "|" + \ str(fb_time) + "|" + \ str(response['duration']) # Remove special characters not allowed in Azure PartitionKey and RowKey response['PartitionKey'] = re.sub("[\ /?#]", "_", response['PartitionKey']) response['RowKey'] = re.sub("[\ /?#]", "_", response['RowKey']) #logging.debug("*** record response: {}".format(response)) try: self.table_service.insert_entity(self.responses_table_name, response) except Exception as err: logging.exception('Error adding response: {}\n\n{}'.format( response, err)) @timer_section("storage.record_feedback") def record_feedback(self, response): fb_time = int(time.time() * 1000) response['PartitionKey'] = response['question_id'] response['RowKey'] = response['type'] + "|" + response[ 'list_id'] + "|" + str(fb_time) # Remove special characters not allowed in Azure PartitionKey and RowKey response['PartitionKey'] = re.sub("[\ /?#]", "_", response['PartitionKey']) response['RowKey'] = re.sub("[\ /?#]", "_", response['RowKey']) logging.debug("*** record feedback: {}".format(response)) try: self.table_service.insert_entity(self.feedbacks_table_name, response) except Exception as err: logging.exception('Error adding feedback: ' + str(err)) @timer_section("storage.update_session") def update_session(self, session_id, data={}): assert session_id is not None assert data['state_id'] is not None properties = { 'PartitionKey': session_id, 'RowKey': "", 'data': encode_dict(data['data']), 'user_id': data['user_id'], 'state_id': data['state_id'], 'valid': data['valid'] } logging.debug( "storage: updating session: {}, valid={}, state_id={}".format( session_id, data['valid'], data['state_id'])) try: self.table_service.insert_or_replace_entity( self.sessions_table_name, properties) except Exception: logging.exception('Error adding to table ' + self.sessions_table_name + ' record: {}'.format(properties)) @timer_section("get_session") def get_session(self, session_id): try: entity = self.table_service.get_entity(self.sessions_table_name, session_id, "") except AzureMissingResourceHttpError: return None # Azurite simulator returns an empty entity instead of exception, so check here if "user_id" not in entity.keys(): return None logging.debug( "storage: loaded session: {}, valid={}, state_id={}".format( session_id, entity.get('valid'), entity.get('state_id'))) # Compatibility: old records don't have state_id, valid if not "state_id" in entity: entity['state_id'] = None if not "valid" in entity: entity["valid"] = True return { # Convert "None" to None, see above # "user_id": entity["user_id"] if (not entity["user_id"] == "None") else None, "user_id": entity["user_id"], "data": decode_dict(entity["data"]), "state_id": entity["state_id"], "valid": entity["valid"] } # Get all user responses from the response table def get_all_user_sessions(self, u_id, from_date=None): req = "(user_id eq '{}')".format(u_id) if from_date: if len(req) > 0: req = req + " and " req = req + "(Timestamp ge datetime'{}')".format( from_date.isoformat()) entries = self.table_service.query_entities(self.sessions_table_name, req) result = [] for row in entries: row["data"] = decode_dict(row["data"]) result.append(row) return result # Get direct user feedback before given date def get_all_user_feedback(self, from_date=None): # Ignore JS and Google errors, only return specific user feedback req = "(type ne 'JS_ERROR') and (type ne 'GOOGLE_ERROR')" if from_date: if len(req) > 0: req = req + " and " req = req + "(Timestamp ge datetime'{}')".format( from_date.isoformat()) entries = self.table_service.query_entities(self.feedbacks_table_name, req) result = [] for row in entries: result.append(row) return result # END - Common methods to implement storage interface ################################################################## # Doesn't really work # def wipe_tables(self): # try: # self.table_service.delete_entity(self.users_table_name, "", "") # except Exception as err: # print('Error wiping table, ' + self.users_table_name + ': ' + str(err)) # try: # self.table_service.delete_entity(self.responses_table_name, "", "") # except Exception as err: # print('Error wiping table, ' + self.responses_table_name + ':' + str(err)) # def delete_all_tables(self): # try: # self.table_service.delete_table(self.users_table_name) # except Exception as err: # logging.exception('Error deleting table, ' + self.users_table_name + ': ' + str(err)) # try: # self.table_service.delete_table(self.responses_table_name) # except Exception as err: # logging.exception('Error deleting table, ' + self.responses_table_name + ':' + str(err)) def get_all_responses(self, user_id=None): if user_id is None: req = "" else: req = "PartitionKey = {}".format(user_id) entries = self.table_service.query_entities(self.responses_table_name, req) return entries def get_all_users(self): entries = self.table_service.query_entities(self.users_table_name, "") return entries def print_all_responses(self, user_id=None): entries = self.get_all_responses(user_id) if user_id is None: print(" USER_ID ", end='') else: print("USER_ID = {}\n".format(user_id)) print( " QUESTION_ID LIST_ID RESPONSE_TYPE TIME DURATION CORRECT INCORRECT QUESTIONS" ) for response in entries: if user_id is None: print("{:^30} ".format(response['question_id']), end='') # UID print("{:^20} ".format(response['user_id']), end='') # QID print("{:^16} ".format(response['list_id']), end='') # List ID print("{:^12} ".format(response['response_type']), end='') # RESPONSE TYPE print("{:^26} ".format( time.strftime("%d-%m-%y %H:%M:%S", time.localtime(int(response['time'])))), end='') # TIME print("{:^16} ".format(response['duration']), end='') # DURATION print("{:^10} ".format(response['correct']), end='') # CORRECT print("{:^10} ".format(response['incorrect']), end='') # INCORRECT print("{:^38} ".format(response['questions'])) # QUESTIONS print("\n") def print_all_users(self): entries = self.get_all_users() print( " USER ID NAME EMAIL LAST ACCESSED REMOTE IP USER AGENT USER LANGUAGE" ) for row in entries: print("{:^30} {:^20} {:^20} {:^20} {:^40} {:^40}".format( row['user_id'], row['name'], time.strftime("%d-%m-%y %H:%M:%S", time.localtime(row['last_accessed'])), row['remote_ip'] if 'remote_ip' in row.keys() else "", row['user_agent'] if 'user_agent' in row.keys() else "", row['user_language'] if 'user_language' in row.keys() else "")) print("\n") def get_question_stats(self, q_id=None, from_date=None): req = "" if q_id: # Remove / from q_id # mq_id = ("".join("fractions/q00022".split("/"))) mq_id = ("".join(q_id.split("/"))) req = req + "((RowKey ge '{}|') and (RowKey lt '{}{}'))".format( mq_id, mq_id, chr(255)) if from_date: if len(req) > 0: req = req + " and " req = req + "(Timestamp ge datetime'{}')".format(from_date) entries = self.table_service.query_entities(self.responses_table_name, req) result = [] for row in entries: if "user_id" not in row.keys() or \ "UNKNOWN" in row["user_id"]: continue result.append(row) return result
HUBS = ["movehub", "cityhub", "technichub", "primehub", "nxt"] print("Building commits...") try: pybricks = git.Repo(PYBRICKS_PATH) except Exception as e: print(f"Repository not found at '{PYBRICKS_PATH}':", e) print("try setting the PYBRICKS_PATH environment variable") sys.exit(1) assert not pybricks.bare, "Repository not found" service = TableService(STORAGE_ACCOUNT, STORAGE_KEY) start_hash = service.get_entity(CI_STATUS_TABLE, "build", "lastHash")["hash"] if start_hash == pybricks.commit(PYBRICKS_BRANCH).hexsha: print("Already up to date.") sys.exit(0) # Process the commits in the tree and log the data for commit in pybricks.iter_commits(f"{start_hash}..{PYBRICKS_BRANCH}"): print(f"trying {commit.hexsha}...") try: entity = service.get_entity(FIRMWARE_SIZE_TABLE, "size", commit.hexsha) # if entity is found but some hubs had null size, redo only those hubs hubs = [h for h in HUBS if entity.get(h) is None] except AzureMissingResourceHttpError: # if there is no entry at all, build all hubs hubs = HUBS
class GetTableData: def __init__(self, connection_string=None, storage_account_name=None, account_key=None, table_name=None, **options): assert (storage_account_name and account_key) or connection_string self.storage_account_name = storage_account_name self.account_key = account_key self.table_name = table_name or "defaulttablepython" self.table_service = TableService(account_name=self.storage_account_name, account_key=self.account_key) \ if self.storage_account_name and self.account_key else None if connection_string: self.connection_string = connection_string self.table_service = TableService( connection_string=self.connection_string) self.differential_column = options.get( 'differential_column') or "Timestamp" self.previous_diff_val = None self.executed_at = dt.utcnow() def get_data_on_partition_and_row_key(self, partition_key, row_key): """ Get a particular entity by passing the partition_key and the row_key of the data :param partition_key: :param row_key: :return: an dict object, containing the data """ return self.table_service.get_entity(self.table_name, partition_key, row_key) def get_data(self): """ Get all the data from the table in a json format :return: """ return [ task for task in self.table_service.query_entities(self.table_name) ] def get_data_with_raw_filter(self, raw_query, select=None): """ get data by using the filter conditions :param select: return the selected attributes :param raw_query: raw query format https://docs.microsoft.com/en-us/rest/api/storageservices/Querying-Tables-and-Entities?redirectedfrom=MSDN :return: """ if isinstance(select, list): select = " , ".join(select) return [ task for task in self.table_service.query_entities( self.table_name, filter=raw_query, select=select) ] @beartype def get_data_sql_interface(self, select: (None, str, list), where: (None, str, list)): """ Parse simple SQL like syntax and convert them into table query syntax :param where: condition :param select: Column names to be selected :return: """ select_ = None filter_ = None if isinstance(select, str): if select != "*": select_ = select if isinstance(select, list): select_ = ",".join(select) if isinstance(where, str): filter_ = sanitize_str(where) if isinstance(where, list): san_filter = [sanitize_str(f) for f in where] filter_ = " and ".join(san_filter) return self.get_data_with_raw_filter(raw_query=filter_, select=select_) def return_differential_data(self, differential_column=None, initially_fetch_data_greater_than_this=None, **options): """ Get incremental data on the specified column. This function will fetch data which inserted after the self.previous_diff_val when the function was previously called. :param differential_column: column on which the differential parameter will be implemented :param initially_fetch_data_greater_than_this: initial value to be fetched :param options: :return: """ self.differential_column = differential_column if differential_column else self.differential_column if initially_fetch_data_greater_than_this and (self.previous_diff_val != initially_fetch_data_greater_than_this)\ and (not self.previous_diff_val): self.previous_diff_val = initially_fetch_data_greater_than_this elif not self.previous_diff_val: self.previous_diff_val = f"{(dt.utcnow() - timedelta(minutes=1)).isoformat()[:-3]+'Z'}" if self.differential_column == "Timestamp": data = self.get_data_sql_interface( select="*", where= f"{self.differential_column} > datetime'{self.previous_diff_val}'" ) else: data = self.get_data_sql_interface( select="*", where=f"{self.differential_column} > {self.previous_diff_val}") if len(data) == 0: return [] pd.set_option("display.max_rows", None, "display.max_columns", None) pandas_df = pd.DataFrame(data) pandas_df.sort_values(by=self.differential_column, inplace=True) if self.differential_column == "Timestamp": self.previous_diff_val = pandas_df[ self.differential_column].iloc[-1].isoformat()[:-9] + 'Z' else: self.previous_diff_val = pandas_df[ self.differential_column].iloc[-1] return pandas_df @beartype def set_alert_on_live_data(self, parameter_name: str, threshold: int, **options): """ Set alert on data by giving parameter name and threshold :return: """ alert = Alert(parameter_name=parameter_name, threshold=threshold, **options) alert.set_alert_on_live_data( diff_data_func=self.return_differential_data, **options)
# define the bucket tgz file path bucket_tgz = bucket + '.tgz' # Research in the Azure Table if this entity exists and if its archiving status is success # If the entity complies with this, archiving is not required and this bucket_id was already archived on this peer or another peer # Otherwise, continue. bucket_is_archived = False record_found = False record_status = None try: with contextlib.redirect_stderr(None): record = table_service.get_entity(AZ_STORAGE_TABLE_NAME, AZ_BLOB_CONTAINER, bucket_id, select='status', timeout=60) record_status = record.status record_found = True except: record_found = False if record_found and record_status in "success": bucket_is_archived = True else: bucket_is_archived = False if bucket_is_archived: print( "This bucket has been archived already, nothing to do and exit 0")
# 要加這行才能順利執行 GPIO.setwarnings(False) # Capture SIGINT for cleanup when the script is aborted def end_read(signal, frame): global continue_reading print "\nCtrl+C captured, ending read." continue_reading = False GPIO.cleanup() # Hook the SIGINT signal.signal(signal.SIGINT, end_read) # Welcome message print "Press Ctrl-C to stop." # 不斷讀取具有是否要開門命令的實體值 while continue_reading: # 查詢實體 task = table_service.get_entity("Projects", "Door", "002") # 偵測是否為可通行的卡,是的話,則value值應為1,亮燈 if task.Value == "1": GPIO.output(LEDPin, GPIO.HIGH) else: GPIO.output(LEDPin, GPIO.LOW) time.sleep(0.5)