def test_constructs_single_query_no_deps(self): code = SqlCode("select company_name, company_id from companies", "companies") repo = Repository() repo.add_queries(code.queries) qg = QueryGenerator(repo) output = qg.generate_query("companies") self.assertEqual(output.lower(), "select company_name, company_id from companies")
def getQueries(allBanners): ''' This function extracts search queries from the banners. :param allBanners: a list of all banners :return: List of all queries, dictionary that maps banners to their queries, and dictionary that maps queries to their to banners ''' if os.path.exists(config.QUERY_LOG_FILE): print("queryLog already exists.") getQueriesFromLogs(allBanners) allQueries, bannerToQueryMap, queryToBannerMap = getQueriesFromLogs( allBanners) t1 = datetime.now() print("Got queries!", len(allQueries), t1) return allQueries, bannerToQueryMap, queryToBannerMap QGen = QueryGenerator() allQueries = [] bannerToQueryMap = QGen.generateForAll(allBanners) queryToBannerMap = {} for banner, query in bannerToQueryMap.items(): if query not in queryToBannerMap.keys(): queryToBannerMap[query] = [banner] else: queryToBannerMap[query].append(banner) allQueries = [query for banner, query in bannerToQueryMap.items()] t1 = datetime.now() print("Got queries!", len(allQueries), t1) logQueries(allQueries) return allQueries, bannerToQueryMap, queryToBannerMap
def export(self, query: Query): qg = QueryGenerator(self.repo) output_sql_text = qg.generate_query(query.name) fields = self.format_fields(query.columns) output_text = self.base.format(query.name.replace("looker_", ""), query.columns[0].name, output_sql_text, fields) with open(self.file_name, "w") as file_open: file_open.write(output_text)
def printresults(): logger.info('{} {} {} {}'.format(dataStartTime.strftime("%Y-%m-%d %H:%M:%S"), dataEndTime.strftime("%Y-%m-%d %H:%M:%S"), runTime, queryPerSec)) line = list() querypermin = queryPerSec * 60 endtime = datetime.now(timezone('UTC')) + timedelta(minutes=runtime) popularitylist = list() while True: tm.sleep(5) time = datetime.now(timezone('UTC')) logger.info("Time: {}".format(time.strftime("%Y-%m-%d %H:%M:%S"))) if time >= endtime: break #Query generated every minute. This is to optimize the overhead of query generation and also because segment granularity is minute newquerylist = list() if filename != "": newquerylist = QueryGenerator.generateQueriesFromFile(dataStartTime, dataEndTime, querypermin, timeAccessGenerator, periodAccessGenerator, filename) elif isbatch == True: newquerylist = QueryGenerator.generateQueries(dataStartTime, dataEndTime, querypermin, timeAccessGenerator, periodAccessGenerator, popularitylist) else: #newquerylist = QueryGenerator.generateQueries(dataStartTime, time, querypermin, timeAccessGenerator, periodAccessGenerator, popularitylist) newquerylist = QueryGenerator.generateQueries(dataStartTime, time, querypermin, timeAccessGenerator, periodAccessGenerator, popularitylist) #print str(newquerylist) for query in newquerylist: print query.interval try: line.append(applyOperation(query, config, logger)) except Exception as inst: logger.info(type(inst)) # the exception instance logger.info(inst.args) # arguments stored in .args logger.info(inst) # __str__ allows args to be printed directly x, y = inst.args logger.info('x =', x) logger.info('y =', y) nextminute = time + timedelta(minutes=1) timediff = (nextminute - datetime.now(timezone('UTC'))).total_seconds() if timediff > 0: tm.sleep(timediff) # yield gen.sleep(timediff) wait_iterator = gen.WaitIterator(*line) while not wait_iterator.done(): try: result = yield wait_iterator.next() except Exception as e: logger.info("Error {} from {}".format(e, wait_iterator.current_future)) else: logger.info("Result {} received from {} at {}".format( result, wait_iterator.current_future, wait_iterator.current_index))
def export(self, query: Query): qg = QueryGenerator(self.repo) output_sql_text = qg.generate_query(query.name) fields = self.format_fields(query.columns) output_text = self.base.format(query.name.replace("looker_", ""), query.columns[0].name, output_sql_text, fields) with open(self.file_name, "w") as file_open: file_open.write(output_text)
def addUser(self, name: str, slackId: str) -> User: addUserQuery = QueryGenerator.generateAddUser(name, slackId) self.users.insert_one(addUserQuery) #TODO return self.findUser(slackId)
def execute_scheme(self): # Instansiera modell model = TimeSeriesPredictionNeuralNet(self.settings) # Importera modellinställningar model.setup() model.compile_model() connection = SQLAConnection() query_generator = QueryGenerator( self.settings.sensors, self.settings.start_date, self.settings.end_date ) # Instansiera data data = NewData(query_generator,connection) data.figure_out_length(model) data.make_new_df_postgres() data.find_discontinuities() data.split_at_discontinuities() data.preprocess(self.settings.normalization) data.train_test_split(self.data_split) model.make_timeseries_dataset(data) #print(data.dfs) # Läs in neuralnät model.load_nn() model.test() model.plot_outliers() model.plot_example()
def threadoperation(start, time, numqueries, timeAccessGenerator, minqueryperiod, maxqueryperiod, periodAccessGenerator, config, logger, x, values): successfulquerytime = 0 successfulquerycount = 0 failedquerytime = 0 failedquerycount = 0 totalquerytime = 0 totalquerycount = 0 endtime = datetime.now() + timedelta(minutes=runtime) while True: if datetime.now() >= endtime: break time = datetime.now(timezone('UTC')) newquerylist = QueryGenerator.generateQueries(start, time, numqueries, timeAccessGenerator, minqueryperiod, maxqueryperiod, periodAccessGenerator); line = applyOperation(newquerylist[0], config,logger) #print line[0:10] if (line[0][0:10] == "Successful"): print line[1].count successfulquerytime += float(line[0][12:]) successfulquerycount += 1 totalquerytime += float(line[0][12:]) totalquerycount += 1 elif (line[0][0:6] == "Failed"): failedquerytime += float(line[0][8:]) failedquerycount += 1 totalquerytime += float(line[0][8:]) totalquerycount += 1 datastructure = [successfulquerytime, successfulquerycount, failedquerytime, failedquerycount, totalquerytime, totalquerycount] values.put(datastructure)
def printresults(): logger.info('{} {} {} {}'.format(start.strftime("%Y-%m-%d %H:%M:%S"), end.strftime("%Y-%m-%d %H:%M:%S"), runtime, queryPerSec)) line = list() querypermin = queryPerSec * 60 endtime = datetime.now(timezone('UTC')) + timedelta(minutes=runtime) popularitylist = list() newquerylist = list() if filename != "": newquerylist = QueryGenerator.generateQueriesFromFile( start, end, querypermin * runtime, timeAccessGenerator, periodAccessGenerator, filename) elif isbatch == True: newquerylist = QueryGenerator.generateQueries( start, end, querypermin * runtime, timeAccessGenerator, periodAccessGenerator, popularitylist)
def findUser(self, slackId: str) -> User: findUserQuery = QueryGenerator.generateFindUser(slackId) userDoc = self.users.find_one(findUserQuery) if not userDoc: return None else: foundUser = User(userDoc["_id"], userDoc["name"], userDoc["slackId"], userDoc["lastUseDate"]) return foundUser
def __init__(self, num_clients, duration, mysql_user, mysql_pass, mysql_host, mysql_port, mysql_socket, database, profile, tables): self.num_clients = num_clients self.duration = duration self.client_list = [] self.client_stats = [] self.pipe_list = [] self.profile = profile self.stats_frequency = 0.0 self.total_clients = 0 self.write_sql_to_disk = False for item in num_clients: self.total_clients += int(item) left_over = 0 if str(duration)[-1] != 's': tduration = int(int(duration) / self.total_clients) left_over = int(duration) - (tduration * self.total_clients) duration = tduration i = 0 for j in range(len(num_clients)): for n in range(int(self.num_clients[j])): self.client_stats.append(ClientQueryStats(str(i))) self.pipe_list.append(multiprocessing.Pipe()) if n == int(self.num_clients[j]) - 1 and left_over > 0: self.client_list.append( QueryGenerator(self.pipe_list[i][1], mysql_user, mysql_pass, mysql_host, mysql_port, mysql_socket, database, tables, duration + left_over, profile[j])) else: self.client_list.append( QueryGenerator(self.pipe_list[i][1], mysql_user, mysql_pass, mysql_host, mysql_port, mysql_socket, database, tables, duration, profile[j])) i += 1
def execute_scheme(self): model = TimeSeriesPredictionNeuralNet(self.settings) model.setup() model.compile_model() connection = SQLAConnection() query_generator = QueryGenerator( self.settings.sensors, self.settings.start_date, self.settings.end_date ) data = Data(query_generator,connection) if self.args.load_dataframe: data.load_dfs(date='2020-11-01') #data.load_extend_dfs(date='2020-11-02') #data.load_extend_dfs(date='2020-11-03') #data.load_extend_dfs(date='2020-11-04') #data.load_extend_dfs(date='2020-11-08') #data.load_extend_dfs(date='2020-11-12') #data.load_extend_dfs(date='2020-11-16') #data.load_extend_dfs(date='2020-11-20') #data.load_extend_dfs(date='2020-11-24') #data.load_extend_dfs(date='2020-11-28') #data.load_extend_dfs(date='2020-12-02') #data.load_extend_dfs(date='2020-12-18') #data.load_extend_dfs(date='2020-12-30') #data.load_extend_dfs(date='2020-12-08') else: data.make_df_postgres() data.find_discontinuities() data.split_at_discontinuities() data.preprocess(self.settings.normalization) data.add_trig() data.train_test_split(self.data_split) model.make_timeseries_dataset(data) model.print_shape() model.plot_example() #model.save_dataset() if self.args.load: model.load_nn() model.train() model.plot_history() model.evaluate() model.save_nn(overwrite=True) model.test() model.plot_outliers() model.plot_example()
def execute_scheme(self): #model = TimeSeriesClassificationNeuralNet(self.settings) #model = TimeSeriesPredictionNeuralNet(self.settings) connection = SQLAConnection() query_generator = QueryGenerator( self.settings.sensors, self.settings.start_date, self.settings.end_date ) report_generator = ReportGenerator(self.settings) link_generator = LinkGenerator(self.settings) data = PostgresData(query_generator,connection) #data.generate_metadata_report(ReportGenerator(self.settings)) #data.load_df(name=self.settings.dataset_name) data.make_df() #data.save_df(name=self.settings.dataset_name) data.find_discontinuities() data.split_at_discontinuities() #data.plot_data() #data.add_temp() data.save_dfs(name=self.settings.start_date)
class LegendLinksGetter: """ Pobiera wyniki wyszukiwania z różnych przeglądarek. """ def __init__(self): """ Konstruktor... """ self.query_generator = QueryGenerator() self.response_gatherer = ResponseGatherer() def search(self, legend_text): """ Szuka w Internetach linków do legend. legend_text jest pełnym tekstem legendy. Metoda zwraca listę wyników. Wynik w ma pola: w.url_title - tytuł w.url - adres url w.snippet - fragment w.engine - wyszukiwarka """ queries = self.query_generator.generate_queries(legend_text) results = self.response_gatherer.get_results(queries) return results
def export(self, query: Query): qg = QueryGenerator(self.repo) output_text = qg.generate_query(query.name) with open(self.file_name, "w") as file_open: file_open.write(output_text)
# print(graph_conf) # graph_conf = [] # for i in graph_size: # for j in range(0, i[2]): # graph_conf.append([i[0], i[1], query_size]) # Export raw graph from Neo4j DB ge = GraphExporter() raw_graph, g1_level, g2_level = ge.export(uri, usename, password, raw_graph_class_node, raw_graph_class_edge) # # # Create new graph (including dumping to file) # # gg = GraphGenerator(raw_graph, g1_level, g2_level, graph_size, graph_level, graph_number) print("Analyze End!") gg = GraphGenerator(raw_graph, (g1_level + 1), (g2_level + 1), graph_conf, base_path, base_path_data, base_path_result) new_graphs, node_id, edge_id = gg.create() print("BOM Generate End!") qg = QueryGenerator(new_graphs, graph_conf, node_id, edge_id, base_path, graph_path, base_path_data, base_path_result) qg.generate() print("Query Generate End!") print( "[INFO] End. (" + str(time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))) + ")") # return
def printresults(): logger.log( STATS, '{} {} {} {} {}'.format(start.strftime("%Y-%m-%d %H:%M:%S"), end.strftime("%Y-%m-%d %H:%M:%S"), runtime, queryPerSec, queryratio)) querypermin = queryPerSec * 60 endtime = datetime.now(timezone('UTC')) + timedelta(minutes=runtime) line = list() popularitylist = list() newquerylist = list() if filename != "": newquerylist = QueryGenerator.generateQueriesFromFile( start, end, querypermin * runtime, timeAccessGenerator, periodAccessGenerator, querytype, queryratio, filename) elif isbatch == True: newquerylist = QueryGenerator.generateQueries( start, end, querypermin * runtime, timeAccessGenerator, periodAccessGenerator, popularitylist, querytype, queryratio, logger) else: #logger.info("Run.py start queryendtime "+str(start)+", "+str(endtime)) queryStartInterval = start queryEndInterval = start + timedelta(minutes=1) for i in range(0, runtime): logger.info("Start generating queries for interval " + str(queryStartInterval) + " - " + str(queryEndInterval)) newquerylist.extend( QueryGenerator.generateQueries( queryStartInterval, queryEndInterval, querypermin, timeAccessGenerator, periodAccessGenerator, popularitylist, querytype, queryratio, logger)) queryEndInterval = queryEndInterval + timedelta(minutes=1) logger.info("Finished generating queries. num queries generated " + str(len(newquerylist))) if filename != "" or isbatch == True: count = 0 time = datetime.now(timezone('UTC')) logger.info("Time: {}".format(time.strftime("%Y-%m-%d %H:%M:%S"))) nextminute = time + timedelta(minutes=1) for query in newquerylist: try: line.append( applyOperation(query, config, brokernameurl, logger)) except Exception as inst: logger.error(type(inst)) # the exception instance logger.error(inst.args) # arguments stored in .args logger.error( inst) # __str__ allows args to be printed directly x, y = inst.args logger.error('x =', x) logger.error('y =', y) count = count + 1 if count >= querypermin: timediff = (nextminute - datetime.now(timezone('UTC'))).total_seconds() if timediff > 0: yield gen.sleep(timediff) count = 0 time = datetime.now(timezone('UTC')) logger.info("Time: {}".format( time.strftime("%Y-%m-%d %H:%M:%S"))) nextminute = time + timedelta(minutes=1) else: # frequency of queries per millisecond queryPerMilliSecond = float(queryPerSec) / 1000 # number of samples spaced by 1 millisecond numSamples = runtime * 60 * 1000 numQueries, querySchedule = genPoissonQuerySchedule( queryPerMilliSecond, numSamples) logger.info("Poisson numQueries = " + str(numQueries)) queryScheduleIdx = 0 count = 0 while count < len(newquerylist): sample = querySchedule[queryScheduleIdx] #logger.info("Poisson sample is "+str(sample[0])+", "+str(sample[1])) if (sample[0] == 0): #logger.info("Sleeping for "+str(sample[1])) yield gen.sleep( float(sample[1]) / 1000) # divide by 1000 to convert it into seconds else: for i in range(0, sample[0]): try: line.append( applyOperation(newquerylist[count], config, brokernameurl, logger)) #applyOperation(newquerylist[count], config, brokernameurl, logger) newquerylist[count].setTxTime(datetime.now()) #logger.info("Running query "+str(sample[0])) except Exception as inst: logger.error(type(inst)) # the exception instance logger.error( inst.args) # arguments stored in .args logger.error( inst ) # __str__ allows args to be printed directly count = count + 1 if count >= len(newquerylist): break queryScheduleIdx = queryScheduleIdx + 1 wait_iterator = gen.WaitIterator(*line) while not wait_iterator.done(): try: result = yield wait_iterator.next() except Exception as e: logger.error("Error {} from {}".format( e, wait_iterator.current_future))
def __init__(self): """ Konstruktor... """ self.query_generator = QueryGenerator() self.response_gatherer = ResponseGatherer()
def execute_scheme(self): #model = TimeSeriesClassificationNeuralNet(self.settings) #model = TimeSeriesPredictionNeuralNet(self.settings) connection = SQLAConnection() query_generator = QueryGenerator( self.settings.sensors, self.settings.start_date, self.settings.end_date ) report_generator = ReportGenerator(self.settings) link_generator = LinkGenerator(self.settings) #data = RegularityData(link_generator,connection) data = AnomalyData(link_generator,connection) #data.generate_metadata_report(ReportGenerator(self.settings)) #data.make_df() #data.save_df(name=self.settings.dataset_name) #data.find_discontinuities() #data.split_at_discontinuities() #data.plot_data() #data.add_temp() #data.save_dfs(name=self.settings.dataset_name) #data.load_dfs(date='2020-11-01') #data.load_extend_dfs(date='2020-11-13') startdate = datetime.strptime('2020-11-01',config.dateformat) data.load_dfs(date=datetime.strftime(startdate,config.dateformat)) dates_ahead = 4 mode = 'while' if mode == 'for': for i in range(dates_ahead): data.load_extend_dfs(date=datetime.strftime(startdate+timedelta(days=i), config.dateformat)) elif mode == 'while': tdate = startdate while tdate.date() != date.today(): try: data.load_extend_dfs(date=datetime.strftime(tdate, config.dateformat)) except FileNotFoundError: pass tdate = tdate+timedelta(days=1) data.purge_empty_dfs() data.preprocess() data.merge_dfs() #data.plot_data() #data.find_correlation() anomaly_settings = AnomalySettings() kmeans_settings = KMeansSettings() start_hour = '00:00:00' end_hour = '23:59:59' data.filter_hours(start_hour,end_hour) data.purge_empty_time_filtered_dfs() #data.plot_filtered_hours(plot_objects=False) data.set_object_settings(anomaly_settings) anomaly_name = f"{startdate}_{mode}_{start_hour}_{end_hour}_{anomaly_settings.anomaly_sensor}_anomaly" print(os.listdir(config.anomaly_path)) print(anomaly_name) if f"{anomaly_name}.json" in os.listdir(config.anomaly_path): data.load_objects(name=f"{anomaly_name}.json") print(f"{anomaly_name} loaded") else: for feature in anomaly_settings.anomaly_sensor: #data.locate_anomalies_filtered_dfs(feature) data.locate_objects_dfs(feature) #data.save_plots(feature) #data.plot_filtered_hours(foi = feature) data.save_objects(name=anomaly_name) kmeans = KMeansClustering(data.objects,kmeans_settings) kmeans.fit_Kmeans() #sensor_prediction = SensorPrediction(data.anomalies,self.settings) data.plot_filtered_hours(foi = 'acc1_ch_x')#,project_anomalies = 'acc1_ch_z') pca = PCAAnomalies(data.objects,self.settings) pca.fit_PCA() pca.save_pca(f'{anomaly_name}_pca') pca.set_labels(kmeans.send_labels()) #pca.get_cov() #anomaly_key, df_number = pca.get_argmax(col='sigma') #data.plot_regularities() pca.plot_components_labels(n_categories = kmeans_settings.n_clusters) pca.scree_plot() pca.plot_hist_pca() #pca.plot_components_3d() pca.plot_components(features = ['Duration','frequency'])
#### LOAD Phase #### print "Pre loading segments and adding to deep storage" segmentlist = RealTimeNode.generateSegments(preloadsegment) deepstorage.extend(segmentlist) segmentrunningcount = len(deepstorage) RealTimeNode.printlist(segmentlist) #### RUN Phase #### for time in xrange(totaltime): #Generating Queries print "Generating Queries" maxquerysize = min(segmentrunningcount, querymaxsize) minquerysize = min(queryminsize, maxquerysize) newquerylist = QueryGenerator.generateQueries( queryperinterval, segmentrunningcount, DistributionFactory.createSegmentDistribution( querysegmentdistribution), minquerysize, maxquerysize, DistributionFactory.createSizeDistribution(querysizedistribution)) Utils.printQueryList(newquerylist) allquerylist.extend(newquerylist) #Routing Queries for strategy in dynamicstrategies: strategy.routeQueries(newquerylist, segmentrunningcount, time) if time % segmentinterval == 0: #Generating Segments indexed starting from 1 print "Generating Segments and adding to deep storage" newsegments = RealTimeNode.generateSegments(segmentsperinterval) RealTimeNode.printlist(newsegments) segmentlist.extend(newsegments)
def addList(self, userId: str, listName: str) -> None: addListQuery = QueryGenerator.generateAddList(userId, listName) self.lists.insert_one(addListQuery)
def addItemToList(self, item: str, list: str) -> None: addItemQuery = QueryGenerator.generateAddItemToList(item, list) self.listItems.insert_one(addItemQuery)
def execute_scheme(self): #model = TimeSeriesClassificationNeuralNet(self.settings) model = TimeSeriesPredictionNeuralNet(self.settings) connection = SQLAConnection() query_generator = QueryGenerator( self.settings.sensors, self.settings.start_date, self.settings.end_date ) report_generator = ReportGenerator(self.settings) link_generator = LinkGenerator(self.settings) data = AnomalyData(link_generator,connection) #data.generate_metadata_report(ReportGenerator(self.settings)) #data.make_df() #data.save_df(name=self.settings.dataset_name) #data.find_discontinuities() #data.split_at_discontinuities() #data.plot_data() #data.add_temp() #data.save_dfs(name=self.settings.dataset_name) #data.load_dfs(date='2020-11-01') #data.load_extend_dfs(date='2020-11-13') startdate = datetime.strptime('2020-11-20',config.dateformat) data.load_dfs(date=datetime.strftime(startdate,config.dateformat)) dates_ahead = 2 mode = 'while' if mode == 'for': for i in range(dates_ahead): data.load_extend_dfs(date=datetime.strftime(startdate+timedelta(days=i), config.dateformat)) elif mode == 'while': tdate = startdate while tdate.date() != date.today(): try: data.load_extend_dfs(date=datetime.strftime(tdate, config.dateformat)) except FileNotFoundError: pass tdate = tdate+timedelta(days=1) data.purge_empty_dfs() data.preprocess() data.merge_dfs() #data.find_correlation() anomaly_settings = AnomalySettings() data.filter_hours('02:00:00','05:00:00') data.set_anomaly_settings(anomaly_settings) for feature in anomaly_settings.anomaly_sensor: #data.locate_anomalies_filtered_dfs(feature) data.locate_anomalies_dfs(feature) #data.save_plots(feature) #data.plot_filtered_hours(foi = feature) #data.plot_filtered_hours(foi = 'acc1_ch_z')#,project_anomalies = 'acc1_ch_z') pca = PCAAnomalies(data.anomalies) pca.fit_PCA() #pca.get_cov() #anomaly_key, df_number = pca.get_argmax(col='sigma') pca.plot_components() pca.scree_plot() pca.plot_hist() pca.plot_components_3d()
if time >= warmuptime: #Generating Queries print "Generating Queries" if changequerydistribution == True and time == changedistributionat: print "Distribution Change" querysegmentdistribution=newquerysegmentdistribution maxqueryperiod = min(time, querymaxsize) minqueryperiod = min(queryminsize, maxqueryperiod) numqueries = queryperinterval; if burstyquery == True and time % burstyqueryinterval == 0: print "Query Burst" numqueries *= burstyquerymultiplier newquerylist = QueryGenerator.generateQueries(time, numqueries, deepstorage, DistributionFactory.createSegmentDistribution(querysegmentdistribution), minqueryperiod, maxqueryperiod, DistributionFactory.createSizeDistribution(querysizedistribution)); Utils.printQueryList(newquerylist) allquerylist.extend(newquerylist) #Routing Queries for strategy in dynamicstrategies: strategy.routeQueries(newquerylist, segmentrunningcount, time) #Placing Segments if time % coordinatorinterval == 0: for strategy in dynamicstrategies: strategy.placeSegments(segmentlist, time, config) segmentlist = [] #Print Statistics for strategy in dynamicstrategies:
print("Segments per Interval: %s" % segmentsperinterval) #### LOAD Phase #### print "Pre loading segments and adding to deep storage" segmentlist = RealTimeNode.generateSegments(preloadsegment) deepstorage.extend(segmentlist) segmentrunningcount = len(deepstorage) RealTimeNode.printlist(segmentlist) #### RUN Phase #### for time in xrange(totaltime): #Generating Queries print "Generating Queries" maxquerysize = min(segmentrunningcount, querymaxsize) minquerysize = min(queryminsize, maxquerysize) newquerylist = QueryGenerator.generateQueries(queryperinterval, segmentrunningcount, DistributionFactory.createSegmentDistribution(querysegmentdistribution), minquerysize, maxquerysize, DistributionFactory.createSizeDistribution(querysizedistribution)); Utils.printQueryList(newquerylist) allquerylist.extend(newquerylist) #Routing Queries for strategy in dynamicstrategies: strategy.routeQueries(newquerylist, segmentrunningcount, time) if time % segmentinterval == 0: #Generating Segments indexed starting from 1 print "Generating Segments and adding to deep storage" newsegments = RealTimeNode.generateSegments(segmentsperinterval) RealTimeNode.printlist(newsegments) segmentlist.extend(newsegments) deepstorage.extend(newsegments) segmentrunningcount = len(deepstorage)
def export(self, query: Query): qg = QueryGenerator(self.repo) output_text = qg.generate_query(query.name) with open(self.file_name, "w") as file_open: file_open.write(output_text)
querysizedistribution = config.getQuerySizeDistribution() queryminsize = config.getQueryMinSize() querymaxsize = config.getQueryMaxSize() historicalnodecount = config.getHistoricalNodeCount() placementstrategy = config.getPlacementStrategy() replicationfactor = config.getReplicationFactor() percentreplicate = config.getPercentReplicate() #Generating Segments indexed starting from 1 print "Generating Segments" segmentlist = RealTimeNode.generateSegments(segmentcount) RealTimeNode.printlist(segmentlist) #Generating Queries print "Generating Queries" querylist = QueryGenerator.generateQueries(querycount, segmentcount, DistributionFactory.createSegmentDistribution(querysegmentdistribution), queryminsize, querymaxsize, DistributionFactory.createSizeDistribution(querysizedistribution)); printQueryList(querylist) ### DRUID COST BASED placementstrategy = "druidcostbased" for replicationfactor in xrange(2, 3): ### Random Routing Stretagy routingstrategy = "random" runExperiment(historicalnodecount, segmentlist, percentreplicate, replicationfactor, querylist, placementstrategy, routingstrategy) ### Connection Count Based Strategy routingstrategy = "chooseleastloaded" runExperiment(historicalnodecount, segmentlist, percentreplicate, replicationfactor, querylist, placementstrategy, routingstrategy) ### RANDOM BALL BASED
self.__print_line__(str(line)) sqlslap_process.wait() def run_scenarios(self, FLAGIN_FLAGOUT=False, no_backend=False, without_query_rules=False, distribution='normal'): self.__apply_query_rules_to_proxy_sql__(FLAGIN_FLAGOUT, no_backend, without_query_rules) self.__load_tests__(distribution) if __name__ == "__main__": query_generator = QueryGenerator() query_rules_generator = QueryRulesGenerator() benchmark = Benchmark(query_generator, query_rules_generator) print('----------------------Normal Distribution-------------------------') print('query rules without flagIN and flagOUT and with backend server') benchmark.run_scenarios(FLAGIN_FLAGOUT=False, no_backend=False, distribution='normal') print('******************************************************************') print('query rules with flagIN and flagOUT and with backend server') benchmark.run_scenarios(FLAGIN_FLAGOUT=True, no_backend=False, distribution='normal') print('******************************************************************') print('query rules without flagIN and flagOUT and without backend server')
def threadoperation(start, time, numqueries, timeAccessGenerator, minqueryperiod, maxqueryperiod, periodAccessGenerator, config, logger, x, values): successfulquerytime = 0 successfulquerycount = 0 failedquerytime = 0 failedquerycount = 0 totalquerytime = 0 totalquerycount = 0 endtime = datetime.now() + dt.timedelta(minutes=runtime) currentSegRank = [] #oldest_timestamp = start.total_seconds() break_flag = 0 #print "16" while True: #while break_flag <1: break_flag = break_flag + 1 #print "1" if datetime.now() >= endtime: break time = datetime.now(timezone('UTC')) #newquerylist = QueryGenerator.generateQueries(start, time, numqueries, timeAccessGenerator, minqueryperiod, maxqueryperiod, periodAccessGenerator); if (len(currentSegRank) == 0): #print "4" y = time - start z = y.total_seconds() x = dt.timedelta(seconds=z) distance = x.total_seconds() for i in range(0, int(round(distance))): timepoint = start + dt.timedelta(0, i) currentSegRank.append(timepoint) else: #y = time.total_seconds() #z = start.total_seconds() #print "2" new_interval = (time - start).total_seconds() #print "time" print time #print "start" print start print new_interval for i in range(0, int(round(new_interval))): samples = randZipf(1 + len(currentSegRank), 1.2, 1) timepoint = start + dt.timedelta(0, i) print "add to rank: index" print samples[0] print "timepoint" print timepoint print "list seg size %d " % len(currentSegRank) #len(currentSegRank) currentSegRank.insert(samples[0], timepoint) print currentSegRank print filename if (filename != ""): #break_flag = 1 newquerylist = QueryGenerator.generateQueriesFromFile( start, time, timeAccessGenerator, minqueryperiod, maxqueryperiod, periodAccessGenerator, filename) #elif(accessdistribution == "dynamiczip"): #newquerylist = QueryGenerator.generateQueries(start, time, numqueries, timeAccessGenerator, minqueryperiod, maxqueryperiod, periodAccessGenerator, currentSegRank) else: #print "3" newquerylist = QueryGenerator.generateQueries( start, time, numqueries, timeAccessGenerator, minqueryperiod, maxqueryperiod, periodAccessGenerator, currentSegRank) line = applyOperation(newquerylist[0], config, logger) print line[0:10] if (line[0][0:10] == "Successful"): print line[1].count successfulquerytime += float(line[0][12:]) successfulquerycount += 1 totalquerytime += float(line[0][12:]) totalquerycount += 1 elif (line[0][0:6] == "Failed"): failedquerytime += float(line[0][8:]) failedquerycount += 1 totalquerytime += float(line[0][8:]) totalquerycount += 1 datastructure = [ successfulquerytime, successfulquerycount, failedquerytime, failedquerycount, totalquerytime, totalquerycount ] values.put(datastructure)
def main(usename, password, raw_graph_class_node, raw_graph_class_edge, query_size, graph_size, socket, uri, graph_path): print( "[INFO] Start. (" + str(time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))) + ")") # Neo4j database configuration # uri = "bolt://127.0.0.1:7687" # usename = "neo4j" # password = "******" # raw_graph_class_node = "item_1" # raw_graph_class_edge = "rel_1" # 使用datetime.now() now = datetime.datetime.now() timestr = str(now.year) + str(now.month) + str(now.day) + str( now.hour) + str(now.minute) + str(now.second) # 需要根据项目所在目录进行改变的 #获取项目的路径 base_path = os.getcwd() base_path_data = os.getcwd() + "/data/data_" + timestr base_path_result = os.getcwd() + "/result/result_" + timestr print("base_path" + base_path) if not os.path.exists(base_path_data): os.makedirs(base_path_data) if not os.path.exists(base_path_result): os.makedirs(base_path_result) #获取服务器上neo4j(图数据库)的路径 # graph_path = "/Users/mark/Documents/MasterLife/GraphDB/neo4j-community-3.4.6" # New graph configuration: graph_conf[size, level, query_num] # query_size = 20 # graph_size = [ [2000, 5, 1] ] # query_size = 1 # graph_size = [ [3000, 7, 1], [6000, 10, 1], [12000, 14, 1], [24000, 20, 1] ] # 2000,5,1&2000, 6, 1 # graph_size = "2000,5,1" _graph_size = graph_size.split("&") graph_conf = [] for i in _graph_size: _i = i.split(",") for j in range(0, int(_i[2])): graph_conf.append([int(_i[0]), int(_i[1]), query_size]) # print(graph_conf) # graph_conf = [] # for i in graph_size: # for j in range(0, i[2]): # graph_conf.append([i[0], i[1], query_size]) # Export raw graph from Neo4j DB socket.wfile.write(("Start to Analyze !<br>").encode()) ge = GraphExporter() raw_graph, g1_level, g2_level = ge.export(uri, usename, password, raw_graph_class_node, raw_graph_class_edge) # # # Create new graph (including dumping to file) # # gg = GraphGenerator(raw_graph, g1_level, g2_level, graph_size, graph_level, graph_number) print("Analyze End!") socket.wfile.write(("Analyze End!!<br>").encode()) gg = GraphGenerator(raw_graph, (g1_level + 1), (g2_level + 1), graph_conf, base_path, base_path_data, base_path_result) new_graphs, node_id, edge_id = gg.create() print("BOM Generate End!") socket.wfile.write(("BOM Generate End!<br>").encode()) qg = QueryGenerator(new_graphs, graph_conf, node_id, edge_id, base_path, graph_path, base_path_data, base_path_result) qg.generate() print("Query Generate End!") socket.wfile.write(("Query Generate End!<br>").encode()) print( "[INFO] End. (" + str(time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))) + ")") return
======= # Instansiera modell >>>>>>> docker-test model = TimeSeriesPredictionNeuralNet(self.settings) # Importera modellinställningar model.setup() <<<<<<< HEAD model.compile_model() if self.args.load_dataset: model.load_dataset() else: connection = SQLAConnection() query_generator = QueryGenerator( self.settings.sensors, self.settings.start_date, self.settings.end_date ) data = Data(query_generator,connection) if self.args.load_dataframe: data.load_dfs(date='2020-11-01') else: data.make_df_postgres() data.find_discontinuities() data.split_at_discontinuities() data.preprocess(self.settings.normalization) #data.fast_fourier_transform() #data.wawelet() #data.STL() data.add_trig() #data.add_temp()