from milvus import Milvus, IndexType, MetricType from utils import gen_collection, insert_data if __name__ == '__main__': collection_name = "milvus_demo_annoy" client = Milvus() # use default host:127.0.0.1, port:19530 client.connect() # create collection: dimension is 128, metric type is L2 gen_collection(client, collection_name, 128, MetricType.L2) # insert 10000 vectors into collection _, _, vectors = insert_data(client, collection_name, 128, 100000) # flush data into disk for persistent storage client.flush([collection_name]) # specify index param index_param = { "n_trees": 50, } # create `ANNOY` index print("Start create index ......") status = client.create_index(collection_name, IndexType.ANNOY, index_param) if status.OK(): print("Create index ANNOY successfully\n") else:
from pymongo import MongoClient from kafka import KafkaConsumer from utils import insert_data # mongo ===== ## Connect cliente = MongoClient('mongodb://*****:*****@localhost:27017/') ## Selecionando um Banco banco = cliente.stagioptr ## collection album = banco.emergency #kafka====== consumer = KafkaConsumer('emergency') print('Iniciando Consumer Emergency!') for message in consumer: insert_data(message, album, topic='emergency')
if __name__ == "__main__": parser = argparse.ArgumentParser(description='modelisation running') parser.add_argument('symbol', type=str, help='') args = parser.parse_args() trade_symbol = args.symbol if len(trade_symbol) != 0 and isinstance(trade_symbol, str) == True: try: df = get_historical_intraday(trade_symbol.upper(), output_format='pandas',token= credentials.token) df = df.fillna(method='ffill') except: pass MYDB = mysql.connector.connect( host=db_credentials.host, user=db_credentials.user, passwd=db_credentials.passwd, database=db_credentials.database ) name = [str(trade_symbol) for i in range(len(df.date.values.tolist()))] df['name'] = name df = df[['date', 'label', 'name', 'high', 'low', 'open', 'close', 'volume', 'numberOfTrades']] VAL = [tuple(x) for x in df.values] SQL = "INSERT INTO raw_stock (date, hour, name, high, low, open, close, volume, numberOfTrades) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)" utils.insert_data(MYDB,SQL,VAL) else: pass
from utils import insert_data from pymongo import MongoClient from kafka import KafkaConsumer from datetime import datetime # mongo ===== # Connect cliente = MongoClient('mongodb://*****:*****@localhost:27017/') # Selecionando um Banco banco = cliente.stagioptr # collection album = banco.sensor # kafka====== consumer = KafkaConsumer('sensor') print('Iniciando Consumer Sensor!') for message in consumer: insert_data(message, album, ["temperatura", "giroscopio", "umidade", "acelerometro"], topic='sensor')
from utils import insert_data from pymongo import MongoClient from kafka import KafkaConsumer # mongo ===== ## Connect cliente = MongoClient('mongodb://*****:*****@localhost:27017/') ## Selecionando um Banco banco = cliente.stagioptr ## collection album = banco.sleep #kafka====== consumer = KafkaConsumer('sleep') print('Iniciando Consumer Sleep!') for message in consumer: insert_data(message, album, topic='sleep')
four_flag = np.random.randint(0, 4, size=500) # 将学生的相关信息组合,使用上面获取的下标向对应的列表获取数据,并保存到数据库里 studentInfos = [] for i in range(0, 500): studentInfo = [ names[i], sex[i], np.random.randint(18, 23), np.random.uniform(1.50, 2.00), np.random.uniform(70, 200), province[province_indexs[i]], parties[four_flag[i]], np.random.randint(100, 2000), havefriend[two_flag[i]], playgame[two_flag[i]] ] studentInfos.append(studentInfo) utils.insert_data(studentInfo) # 将所有学生信息写出为.csv文件 file_csv = codecs.open("studentInfo.csv", 'w+', 'utf-8') writer = csv.writer(file_csv, delimiter=' ', quotechar=' ', quoting=csv.QUOTE_MINIMAL) writer.writerow( ["姓名", "性别", "年龄", "身高", "体重", "省份", "党派", "消费", "有无男/女朋友", "是否打游戏"]) for studentInfo in studentInfos: writer.writerow(studentInfo) # 运行该.py,完成数据获取和保存 if __name__ == '__main__': print("完成数据集获取,并保存到MySQL数据库")
from utils import insert_data from pymongo import MongoClient from kafka import KafkaConsumer # mongo ===== ## Connect cliente = MongoClient('mongodb://*****:*****@localhost:27017/') ## Selecionando um Banco banco = cliente.stagioptr ## collection album = banco.feeding #kafka====== consumer = KafkaConsumer('feeding') print('Iniciando Consumer Feeding!') for message in consumer: insert_data(message, album, topic='feeding')
from utils import insert_data from pymongo import MongoClient from kafka import KafkaConsumer # mongo ===== ## Connect cliente = MongoClient('mongodb://*****:*****@localhost:27017/') ## Selecionando um Banco banco = cliente.stagioptr ## collection album = banco.emotional #kafka====== consumer = KafkaConsumer('emotional') print('Iniciando Consumer Emotional!') for message in consumer: insert_data(message, album, topic='emotional')