def main(event: func.EventHubEvent): handler = LoggingHandler(os.environ['APPINSIGHTS_INSTRUMENTATIONKEY']) logging.basicConfig(handlers=[ handler ], format='%(levelname)s: %(message)s', level=logging.DEBUG) tc = TelemetryClient(os.environ['APPINSIGHTS_INSTRUMENTATIONKEY']) tc.track_event("Incoming event") tc.flush() logging.info('Function triggered to process a message: %s', event) logging.info(' body: %s', event.get_body()) logging.info(' EnqueuedTimeUtc: %s', event.enqueued_time) logging.info(' SequenceNumber: %s', event.sequence_number) logging.info(' Offset: %s', event.offset) logging.info(' Partition: %s', event.partition_key) logging.info(' Metadata: %s', event.iothub_metadata) table_service = TableService(connection_string=os.environ['AzureTableConnectionString']) for datapoint in json.loads(event.get_body()): # Expected data format: # {"timestamp": 1564598054, "deviceid": "Node1", "scale": 2, "temperature": 1.1,"weight": 10000} if datapoint is not None and 'deviceid' in datapoint and \ 'timestamp' in datapoint and 'scale' in datapoint and \ 'weight' in datapoint: logging.debug(' datapoint: %s', (datapoint)) # deviceid is used as partition key. # {timestamp}-{scale} is used as RowKey # timestamp and scale number are duplicated as an int columns # to keep them searchable. The rest of the datapoint elements # are added as columns as well. history = {} history['PartitionKey'] = datapoint.pop('deviceid') history['RowKey'] = str(datapoint['timestamp']) + '-' + str(datapoint['scale']) history.update(datapoint.items()) logging.debug('history: %s' % (history)) table_service.insert_entity(TABLE_NAME_HISTORICAL_DATA, history) logging.info('Added historical table data: %s', (history)) # Touch/create the row in the config table for each reported scale with latest weight configupdate = {} configupdate['PartitionKey'] = history['PartitionKey'] configupdate['RowKey'] = str(history['scale']) configupdate['weight'] = history['weight'] if 'temperature' in history: configupdate['temperature'] = history['temperature'] logging.info('config update: %s' % (configupdate)) logging.info('Writing to table: %s' % (TABLE_NAME_CONFIGURATION)) table_service.insert_or_merge_entity(TABLE_NAME_CONFIGURATION, configupdate) logging.info('Updated configuration table entry: %s', (configupdate)) else: logging.info(' Invalid datapoint: %s', (datapoint))
def putTableEntity(data): ''' ''' try: # Get Azure Storage Table connection logging.info('Creating Azure Storage Table') table_service = TableService( connection_string=os.environ['SA_CONNX_STRING']) try: logging.info('Checking for existing Azure Storage Table') # Check for table, if none then create new table table_service.create_table(os.environ['SA_TABLE_INSIGHTS'], fail_on_exist=True) logging.info('Success: New table created') except Exception as e: logging.info('Failed: Table already exists {0}'.format(e)) for entity in data: try: # Create unique row key row_key = '{0}_{1}_{2}'.format(entity['vi_file_name'], entity['vi_feature_type'], entity['vi_feature']) task = { 'PartitionKey': 'examplekey', 'RowKey': row_key, 'FileName': entity['vi_file_name'], 'SourceLanguage': entity['vi_source_language'], 'FeatureType': entity['vi_feature_type'], 'Feature': entity['vi_feature'], 'ConfidenceScore': entity['vi_confidence_score'] } # Write entry to Azure Storage Table table_service.insert_or_merge_entity( os.environ['SA_TABLE_INSIGHTS'], task) except Exception as e: logging.info( 'Failed: Put entity to Azure Storage Table {0}'.format(e)) logging.info('Success: Put entity to Azure Storage Table') except Exception as e: logging.info( 'Failed: Put entities to Azure Storage Table {0}'.format(e))
def putTableEntity(video_id, video_name, video_path, insights_path, video_url): ''' ''' try: logging.info('Putting new entity to Azure Storage Table') # Get Azure Storage Table connection table_service = TableService( connection_string=os.environ['SA_CONNX_STRING']) try: logging.info('Checking for existing Azure Storage Table') # Check for table, if none then create new table table_service.create_table(os.environ['SA_TABLE_TRACKER'], fail_on_exist=True) logging.info('Success: New table created') except Exception as e: logging.info('Failed: Table already exists {0}'.format(e)) task = { 'PartitionKey': 'examplekey', 'RowKey': video_id, 'VideoIndexerId': video_id, 'VideoName': video_name, 'VideoPath': video_path, 'VideoUrl': video_url, 'InsightsPath': insights_path, 'State': 'Processed' } # Write entry to Azure Storage Table table_service.insert_or_merge_entity(os.environ['SA_TABLE_TRACKER'], task) logging.info('Success: Put entity to Azure Storage Table') except Exception as e: logging.info('Failed: Put entity to Azure Storage Table {0}'.format(e))
pybricks.submodule("micropython").module().git.submodule( "update", "--init", "lib/stm32lib") if args.hub == "primehub": pybricks.git.submodule("update", "--init", "--checkout", "lib/btstack") if args.hub == "nxt": pybricks.git.submodule("update", "--init", "--checkout", "bricks/nxt/nxt-firmware-drivers") # build the firmware subprocess.check_call([ "make", "-C", os.path.join(PYBRICKS_PATH, "bricks", args.hub), "clean", "build/firmware.bin", "all", ]) # upload firmware size bin_path = os.path.join(PYBRICKS_PATH, "bricks", args.hub, "build", "firmware.bin") size = os.path.getsize(bin_path) service.insert_or_merge_entity( FIRMWARE_SIZE_TABLE, { "PartitionKey": "size", "RowKey": commit.hexsha, args.hub: size, }, )
pybricks.git.submodule("update", "lib/libfixmath") if args.hub in ["cityhub", "movehub", "technichub", "primehub"]: pybricks.submodule("micropython").module().git.submodule( "update", "lib/stm32lib") if args.hub == "nxt": pybricks.git.submodule("update", "--checkout", "bricks/nxt/nxt-firmware-drivers") # build the firmware subprocess.check_call([ "make", "-C", os.path.join(PYBRICKS_PATH, "bricks", args.hub), "clean", "build/firmware.bin", "all", ]) # upload firmware size bin_path = os.path.join(PYBRICKS_PATH, "bricks", args.hub, "build", "firmware.bin") size = os.path.getsize(bin_path) service.insert_or_merge_entity( TABLE_NAME, { "PartitionKey": "size", "RowKey": commit.hexsha, args.hub: size, }, )
class DataControl: def __init__(self): self.table_service = TableService(account_name='toonestorage01', account_key='sIo3TKwG40eH2a9MpjZdGgWwetkGDV3NcgFhwZN2sFerhrj3kWLTiKQO3wGO7bd9sjmGoBnPl2CbqrIJcsnG8g==') #table_service = TableService(connection_string='DefaultEndpointsProtocol=https;AccountName=toonestorage01;AccountKey=sIo3TKwG40eH2a9MpjZdGgWwetkGDV3NcgFhwZN2sFerhrj3kWLTiKQO3wGO7bd9sjmGoBnPl2CbqrIJcsnG8g==;EndpointSuffix=core.windows.net') #SOURCE_TABLE = "stockday" ########################################################################################################################### def set_table(self, tabletype, tooneday): if tabletype == 'D': table_name = 'stock' + tabletype + tooneday #오늘 날짜 분석 테이블이 존재 하면 삭제 하고 새로 생성 / Azure Table은 삭제 후 재 생성 불가 (delay 발생) print(table_name) if self.table_service.exists(table_name) == False: self.table_service.create_table(table_name) print(">> table created : " + table_name ) else : print(">> table exists " + table_name) elif tabletype =='5M': print(">>> 5M table") return table_name def set_target(self, targetdf): prep_df = preprocess(targetdf) print ( prep_df ) ########################################################################################################################### def set_stock_day(self, targettable, stockdf): # print('>> insert data to azure table') ''' for data in stockdata: task = {'PartitionKey': 'tasksSeattle', 'RowKey': '001','description': 'Take out the trash', 'priority': 200} self.table_service.insert_entity('stockday', task) ''' # dataframe에서 partitionkey 및 row_id 로 칼럼명 변경 필요 # key 값을 두개만 가질 수 있음 # particionkey = code / date = row_id #print (stockdf.head()) stockdf_table = stockdf.rename(columns={"code": "PartitionKey", "date": "RowKey"}) #print (stockdf_table) for index, row in stockdf_table.iterrows(): #print(row) #print(row['PartitionKey']) #print(">> start row") #print(row) task = Entity() task.PartitionKey = row.to_dict()['PartitionKey'] task.RowKey = str(row.to_dict()['RowKey']) task.open = row.to_dict()['open'] task.high = row.to_dict()['high'] task.low = row.to_dict()['low'] task.close = row.to_dict()['close'] task.volume = row.to_dict()['volume'] self.table_service.insert_or_merge_entity(targettable, task, timeout=None) #print('>> end set stockday') def get_stock_day(self, code): print('>> start get data ') # 여러 데이타 추출 target_table = 'stockD' + time.strftime('%Y%m%d') filter_target = "PartitionKey eq '" + code+ "'" rows = self.table_service.query_entities( target_table , filter=filter_target, select='PartitionKey,RowKey,open,high,low,volume,close') df_stock_day = pd.DataFrame(rows) #print(df_stock_day.head()) return df_stock_day def set_stock_min(self, stockdf): # print('>> insert data to azure table') #print (stockdf.head()) stockdf_table = stockdf.rename(columns={"code": "PartitionKey", "date": "RowKey"}) #print (stockdf_table) stockdf_table = stockdf_table.astype({"time":str, "RowKey": str}) # date column을 time 컬럼과 합성 if len(str(stockdf_table["time"])) <= 3: stockdf_table["time"] = '0' + str(stockdf_table["time"]) print(stockdf_table.head()) stockdf_table["RowKey"] = stockdf_table["RowKey"] + stockdf_table["time"] print(stockdf_table.head()) stockdf_last = pd.DataFrame() stockdf_last = stockdf_table[ ['PartitionKey', 'RowKey', 'time', 'open', 'high', 'low', 'close', 'volume'] ] print(stockdf_last) for index, row in stockdf_last.iterrows(): task = Entity() task.PartitionKey = row.to_dict()['PartitionKey'] task.RowKey = str(row.to_dict()['RowKey']) task.time = row.to_dict()['time'] task.open = row.to_dict()['open'] task.high = row.to_dict()['high'] task.low = row.to_dict()['low'] task.close = row.to_dict()['close'] task.volume = row.to_dict()['volume'] #print(task) self.table_service.insert_or_merge_entity('stockM', task, timeout=None) ########################################################################################################################### def get_max_date(self, stockcode): filter_str = "PartitionKey eq '" + stockcode +"'" rows = self.table_service.query_entities( 'stockday', filter=filter_str, select='open,close') #print (rows.RowKey) for row in rows: print (row) return row def get_max_time(self, stockcode): filter_str = "RowKey eq '" + stockcode +"'" rows = self.table_service.query_entities( 'stockM', filter=filter_str, select='open,close') #print (rows.RowKey) for row in rows: print (row) return row def set_target_stock(self,df_target): # ['price','volume', 'per','eps'] df_target["date"]=time.strftime('%Y%m%d') stockdf_table = df_target.rename(columns={"date": "PartitionKey", "code": "RowKey"}) for index, row in stockdf_table.iterrows(): #print(row) #print(row['PartitionKey']) #print(">> start row") #print(row) task = Entity() task.PartitionKey = row.to_dict()['PartitionKey'] task.RowKey = str(row.to_dict()['RowKey']) task.price = row.to_dict()['price'] task.volume = row.to_dict()['volume'] task.per = row.to_dict()['per'] task.eps = row.to_dict()['eps'] self.table_service.insert_or_merge_entity('stocktarget', task) print(">> set target stock..." + str(row.to_dict()['RowKey']) ) def get_target_stock(self): filter_target = "PartitionKey eq '" + time.strftime('%Y%m%d')+ "'" rows = self.table_service.query_entities( 'stocktarget', filter= filter_target, select='RowKey, status') #print (rows.RowKey) df_target = pd.DataFrame(rows) return df_target
def create_file_record(url, unique_id, partition_key, short_id, name, extension, relative_path, exifString, xmpString, url_list, md5, sha256, instance_name, account_key): start = time.time() utcnow = datetime.datetime.utcnow().isoformat() # Each ingested (and successfully processed) file has a unique record containing # information, list of previews, file_record = { 'PartitionKey': partition_key, # using tree structure for partition key a good idea? #possiblybadidea #possiblygoodidea 'RowKey': short_id, # using unique file name for key a good idea? #badidea #mustbeuniqueinpartition 'uid': unique_id, # globally uniqueId 'url': url, # master blob url 'name': name, # filename 'ext' : extension, # file extension 'path' : relative_path, # path / folder file lives in 'it': utcnow, # ingestion_time 'pvs' : json.dumps(url_list), # json list of preview urls 'md5' : md5, # md5 checksum of total file binary data at ingestion time 'sha256' : sha256, # sha256 checksum of total file binary data at ingestion time 'exif' : exifString, # exif dumped as json by imagemagick 'xmp' : xmpString, # if exif identified APP1 data, xmp dump in xml by imagemagick 'created_time' : utcnow, # file creation time, using now, TODO: pick up file metadata if provided in upload 'modified_time' : utcnow # file mod time, using now, TODO: pick up file metadata if provided in upload } table_service = TableService(account_name=instance_name, account_key=account_key) table_service.insert_or_replace_entity('files', file_record) print "file_record inserted in {} sec".format(time.time()-start) # Change record to folder facing # TODO: Strip large metadata blocks and keep info needed for UIs file_record["PartitionKey"] = relative_path.replace("/", "%2F") file_record['item_type'] = 'file' table_service.insert_or_replace_entity('folders', file_record) # Ensure we have folder records for the entire path # TODO: Optimization; Check if the final folder exists, if so, skip step (we know all higher level paths have been created too) folder_struct = relative_path.split("/") # partition keys cannot have / in them, this is the best I can come up with atm folder_struct[0] = "%2F" # path starts with slash, will have empty slot first, replace with / last_folder = folder_struct[0] # weird exception case, root refers to itself as parent, but easy to check for later for folder in folder_struct: if len(folder) == 0: # ignore empty paths, tolerate e.g. folder1//folder2/folder3/ continue folder_record = { 'PartitionKey': last_folder, 'RowKey': folder, 'created_time': utcnow, 'modified_time': utcnow, 'nf_flag': True, 'nf_time': utcnow, 'item_type': 'folder' } if len(folder) > 3: # special handling of root last_folder = last_folder + "%2F" + folder # if folder already exist, we will fail, remove the creation properties and # try a merge operation (that should work unless service is down) try: table_service.insert_entity('folders', folder_record) except: folder_record.pop('created_time') table_service.insert_or_merge_entity('folders', folder_record)
class Storage_az_table(): ################################################################## # BEGIN - Common methods to implement storage interface def __init__(self): connection_string = os.environ['SHKOLA_AZ_TABLE_CONN_STR'] self.table_service = TableService(connection_string=connection_string) self.default_partition_key = "USER" self.users_table_name = 'users' self.responses_table_name = 'responses' self.sessions_table_name = 'sessions' self.feedbacks_table_name = 'feedbacks' tables = [ self.users_table_name, self.responses_table_name, self.sessions_table_name, self.feedbacks_table_name ] existing_tables = list( map(lambda x: x.name, self.table_service.list_tables())) for table in tables: if table not in existing_tables: self.table_service.create_table(table) def get_user(self, user_id): partition_key = self.default_partition_key try: entity = self.table_service.get_entity(self.users_table_name, partition_key, user_id) except AzureMissingResourceHttpError: return None entity["user_id"] = user_id return entity def update_user(self, user_id, name=None, remote_ip=None, user_agent=None, picture=None, user_language=None, selected_language=None, last_accessed=None): properties = dict() # Nothing better at the moment: properties['PartitionKey'] = self.default_partition_key properties['RowKey'] = user_id properties['user_id'] = user_id properties["name"] = name properties["remote_ip"] = remote_ip properties["user_agent"] = user_agent properties["picture"] = picture properties["user_language"] = user_language if selected_language: properties["selected_language"] = selected_language properties["last_accessed"] = last_accessed logging.debug("azure table update_user %s: %s", str(name), str(properties)) try: self.table_service.insert_or_merge_entity(self.users_table_name, properties) except Exception: logging.exception('Error adding to table ' + self.users_table_name + ' record: {}'.format(properties)) def update_selected_language(self, user_id, selected_language): properties = dict() # Nothing better at the moment: properties['PartitionKey'] = self.default_partition_key properties['RowKey'] = user_id properties['user_id'] = user_id properties["selected_language"] = selected_language logging.debug("azure table update_user_langauge %s: %s", str(user_id), str(properties)) try: self.table_service.insert_or_merge_entity(self.users_table_name, properties) except Exception: logging.exception('Error adding to table ' + self.users_table_name + ' record: {}'.format(properties)) def insert_user_id(self, user_id): self.update_user(user_id) return user_id # Get all user responses from the response table def get_all_user_results(self, u_id, from_date=None): req = "(PartitionKey eq '{}')".format(u_id) if from_date: if len(req) > 0: req = req + " and " req = req + "(Timestamp ge datetime'{}')".format(from_date) entries = self.table_service.query_entities(self.responses_table_name, req) result = [] for row in entries: result.append(row) return result # Update user info with the latest <stats>, and with # <stats_time> being the time of the latest stats def update_user_stats(self, user_id, stats, stats_time): properties = dict() # Nothing better at the moment: properties['PartitionKey'] = self.default_partition_key properties['RowKey'] = user_id properties['user_id'] = user_id properties['stats'] = encode_dict(stats) properties['stats_time'] = stats_time try: self.table_service.merge_entity(self.users_table_name, properties) except Exception: logging.exception('Error updating results for user ' + user_id + ' record: {}'.format(properties)) @timer_section("storage.record_response") def record_response(self, response): fb_time = int(time.time() * 1000) response['PartitionKey'] = response['user_id'] response['RowKey'] = response['question_id'] + "|" + \ str(response['attempt']) + "|" + \ str(fb_time) + "|" + \ str(response['duration']) # Remove special characters not allowed in Azure PartitionKey and RowKey response['PartitionKey'] = re.sub("[\ /?#]", "_", response['PartitionKey']) response['RowKey'] = re.sub("[\ /?#]", "_", response['RowKey']) #logging.debug("*** record response: {}".format(response)) try: self.table_service.insert_entity(self.responses_table_name, response) except Exception as err: logging.exception('Error adding response: {}\n\n{}'.format( response, err)) @timer_section("storage.record_feedback") def record_feedback(self, response): fb_time = int(time.time() * 1000) response['PartitionKey'] = response['question_id'] response['RowKey'] = response['type'] + "|" + response[ 'list_id'] + "|" + str(fb_time) # Remove special characters not allowed in Azure PartitionKey and RowKey response['PartitionKey'] = re.sub("[\ /?#]", "_", response['PartitionKey']) response['RowKey'] = re.sub("[\ /?#]", "_", response['RowKey']) logging.debug("*** record feedback: {}".format(response)) try: self.table_service.insert_entity(self.feedbacks_table_name, response) except Exception as err: logging.exception('Error adding feedback: ' + str(err)) @timer_section("storage.update_session") def update_session(self, session_id, data={}): assert session_id is not None assert data['state_id'] is not None properties = { 'PartitionKey': session_id, 'RowKey': "", 'data': encode_dict(data['data']), 'user_id': data['user_id'], 'state_id': data['state_id'], 'valid': data['valid'] } logging.debug( "storage: updating session: {}, valid={}, state_id={}".format( session_id, data['valid'], data['state_id'])) try: self.table_service.insert_or_replace_entity( self.sessions_table_name, properties) except Exception: logging.exception('Error adding to table ' + self.sessions_table_name + ' record: {}'.format(properties)) @timer_section("get_session") def get_session(self, session_id): try: entity = self.table_service.get_entity(self.sessions_table_name, session_id, "") except AzureMissingResourceHttpError: return None # Azurite simulator returns an empty entity instead of exception, so check here if "user_id" not in entity.keys(): return None logging.debug( "storage: loaded session: {}, valid={}, state_id={}".format( session_id, entity.get('valid'), entity.get('state_id'))) # Compatibility: old records don't have state_id, valid if not "state_id" in entity: entity['state_id'] = None if not "valid" in entity: entity["valid"] = True return { # Convert "None" to None, see above # "user_id": entity["user_id"] if (not entity["user_id"] == "None") else None, "user_id": entity["user_id"], "data": decode_dict(entity["data"]), "state_id": entity["state_id"], "valid": entity["valid"] } # Get all user responses from the response table def get_all_user_sessions(self, u_id, from_date=None): req = "(user_id eq '{}')".format(u_id) if from_date: if len(req) > 0: req = req + " and " req = req + "(Timestamp ge datetime'{}')".format( from_date.isoformat()) entries = self.table_service.query_entities(self.sessions_table_name, req) result = [] for row in entries: row["data"] = decode_dict(row["data"]) result.append(row) return result # Get direct user feedback before given date def get_all_user_feedback(self, from_date=None): # Ignore JS and Google errors, only return specific user feedback req = "(type ne 'JS_ERROR') and (type ne 'GOOGLE_ERROR')" if from_date: if len(req) > 0: req = req + " and " req = req + "(Timestamp ge datetime'{}')".format( from_date.isoformat()) entries = self.table_service.query_entities(self.feedbacks_table_name, req) result = [] for row in entries: result.append(row) return result # END - Common methods to implement storage interface ################################################################## # Doesn't really work # def wipe_tables(self): # try: # self.table_service.delete_entity(self.users_table_name, "", "") # except Exception as err: # print('Error wiping table, ' + self.users_table_name + ': ' + str(err)) # try: # self.table_service.delete_entity(self.responses_table_name, "", "") # except Exception as err: # print('Error wiping table, ' + self.responses_table_name + ':' + str(err)) # def delete_all_tables(self): # try: # self.table_service.delete_table(self.users_table_name) # except Exception as err: # logging.exception('Error deleting table, ' + self.users_table_name + ': ' + str(err)) # try: # self.table_service.delete_table(self.responses_table_name) # except Exception as err: # logging.exception('Error deleting table, ' + self.responses_table_name + ':' + str(err)) def get_all_responses(self, user_id=None): if user_id is None: req = "" else: req = "PartitionKey = {}".format(user_id) entries = self.table_service.query_entities(self.responses_table_name, req) return entries def get_all_users(self): entries = self.table_service.query_entities(self.users_table_name, "") return entries def print_all_responses(self, user_id=None): entries = self.get_all_responses(user_id) if user_id is None: print(" USER_ID ", end='') else: print("USER_ID = {}\n".format(user_id)) print( " QUESTION_ID LIST_ID RESPONSE_TYPE TIME DURATION CORRECT INCORRECT QUESTIONS" ) for response in entries: if user_id is None: print("{:^30} ".format(response['question_id']), end='') # UID print("{:^20} ".format(response['user_id']), end='') # QID print("{:^16} ".format(response['list_id']), end='') # List ID print("{:^12} ".format(response['response_type']), end='') # RESPONSE TYPE print("{:^26} ".format( time.strftime("%d-%m-%y %H:%M:%S", time.localtime(int(response['time'])))), end='') # TIME print("{:^16} ".format(response['duration']), end='') # DURATION print("{:^10} ".format(response['correct']), end='') # CORRECT print("{:^10} ".format(response['incorrect']), end='') # INCORRECT print("{:^38} ".format(response['questions'])) # QUESTIONS print("\n") def print_all_users(self): entries = self.get_all_users() print( " USER ID NAME EMAIL LAST ACCESSED REMOTE IP USER AGENT USER LANGUAGE" ) for row in entries: print("{:^30} {:^20} {:^20} {:^20} {:^40} {:^40}".format( row['user_id'], row['name'], time.strftime("%d-%m-%y %H:%M:%S", time.localtime(row['last_accessed'])), row['remote_ip'] if 'remote_ip' in row.keys() else "", row['user_agent'] if 'user_agent' in row.keys() else "", row['user_language'] if 'user_language' in row.keys() else "")) print("\n") def get_question_stats(self, q_id=None, from_date=None): req = "" if q_id: # Remove / from q_id # mq_id = ("".join("fractions/q00022".split("/"))) mq_id = ("".join(q_id.split("/"))) req = req + "((RowKey ge '{}|') and (RowKey lt '{}{}'))".format( mq_id, mq_id, chr(255)) if from_date: if len(req) > 0: req = req + " and " req = req + "(Timestamp ge datetime'{}')".format(from_date) entries = self.table_service.query_entities(self.responses_table_name, req) result = [] for row in entries: if "user_id" not in row.keys() or \ "UNKNOWN" in row["user_id"]: continue result.append(row) return result