def test_constructs_single_query_no_deps(self):
code = SqlCode("select company_name, company_id from companies", "companies")
repo = Repository()
repo.add_queries(code.queries)
qg = QueryGenerator(repo)
output = qg.generate_query("companies")
self.assertEqual(output.lower(), "select company_name, company_id from companies")
def getQueries(allBanners):
'''
This function extracts search queries from the banners.
:param allBanners: a list of all banners
:return: List of all queries, dictionary that maps banners to their queries,
and dictionary that maps queries to their to banners
'''
if os.path.exists(config.QUERY_LOG_FILE):
print("queryLog already exists.")
getQueriesFromLogs(allBanners)
allQueries, bannerToQueryMap, queryToBannerMap = getQueriesFromLogs(
allBanners)
t1 = datetime.now()
print("Got queries!", len(allQueries), t1)
return allQueries, bannerToQueryMap, queryToBannerMap
QGen = QueryGenerator()
allQueries = []
bannerToQueryMap = QGen.generateForAll(allBanners)
queryToBannerMap = {}
for banner, query in bannerToQueryMap.items():
if query not in queryToBannerMap.keys():
queryToBannerMap[query] = [banner]
else:
queryToBannerMap[query].append(banner)
allQueries = [query for banner, query in bannerToQueryMap.items()]
t1 = datetime.now()
print("Got queries!", len(allQueries), t1)
logQueries(allQueries)
return allQueries, bannerToQueryMap, queryToBannerMap
def export(self, query: Query):
qg = QueryGenerator(self.repo)
output_sql_text = qg.generate_query(query.name)
fields = self.format_fields(query.columns)
output_text = self.base.format(query.name.replace("looker_", ""), query.columns[0].name, output_sql_text, fields)
with open(self.file_name, "w") as file_open:
file_open.write(output_text)
def printresults():
logger.info('{} {} {} {}'.format(dataStartTime.strftime("%Y-%m-%d %H:%M:%S"), dataEndTime.strftime("%Y-%m-%d %H:%M:%S"), runTime, queryPerSec))
line = list()
querypermin = queryPerSec * 60
endtime = datetime.now(timezone('UTC')) + timedelta(minutes=runtime)
popularitylist = list()
while True:
tm.sleep(5)
time = datetime.now(timezone('UTC'))
logger.info("Time: {}".format(time.strftime("%Y-%m-%d %H:%M:%S")))
if time >= endtime:
break
#Query generated every minute. This is to optimize the overhead of query generation and also because segment granularity is minute
newquerylist = list()
if filename != "":
newquerylist = QueryGenerator.generateQueriesFromFile(dataStartTime, dataEndTime, querypermin, timeAccessGenerator, periodAccessGenerator, filename)
elif isbatch == True:
newquerylist = QueryGenerator.generateQueries(dataStartTime, dataEndTime, querypermin, timeAccessGenerator, periodAccessGenerator, popularitylist)
else:
#newquerylist = QueryGenerator.generateQueries(dataStartTime, time, querypermin, timeAccessGenerator, periodAccessGenerator, popularitylist)
newquerylist = QueryGenerator.generateQueries(dataStartTime, time, querypermin, timeAccessGenerator, periodAccessGenerator, popularitylist)
#print str(newquerylist)
for query in newquerylist:
print query.interval
try:
line.append(applyOperation(query, config, logger))
except Exception as inst:
logger.info(type(inst)) # the exception instance
logger.info(inst.args) # arguments stored in .args
logger.info(inst) # __str__ allows args to be printed directly
x, y = inst.args
logger.info('x =', x)
logger.info('y =', y)
nextminute = time + timedelta(minutes=1)
timediff = (nextminute - datetime.now(timezone('UTC'))).total_seconds()
if timediff > 0:
tm.sleep(timediff)
# yield gen.sleep(timediff)
wait_iterator = gen.WaitIterator(*line)
while not wait_iterator.done():
try:
result = yield wait_iterator.next()
except Exception as e:
logger.info("Error {} from {}".format(e, wait_iterator.current_future))
else:
logger.info("Result {} received from {} at {}".format(
result, wait_iterator.current_future,
wait_iterator.current_index))
def export(self, query: Query):
qg = QueryGenerator(self.repo)
output_sql_text = qg.generate_query(query.name)
fields = self.format_fields(query.columns)
output_text = self.base.format(query.name.replace("looker_", ""),
query.columns[0].name, output_sql_text,
fields)
with open(self.file_name, "w") as file_open:
file_open.write(output_text)
def addUser(self, name: str, slackId: str) -> User:
addUserQuery = QueryGenerator.generateAddUser(name, slackId)
self.users.insert_one(addUserQuery)
#TODO
return self.findUser(slackId)
def execute_scheme(self):
# Instansiera modell
model = TimeSeriesPredictionNeuralNet(self.settings)
# Importera modellinställningar
model.setup()
model.compile_model()
connection = SQLAConnection()
query_generator = QueryGenerator(
self.settings.sensors,
self.settings.start_date,
self.settings.end_date
)
# Instansiera data
data = NewData(query_generator,connection)
data.figure_out_length(model)
data.make_new_df_postgres()
data.find_discontinuities()
data.split_at_discontinuities()
data.preprocess(self.settings.normalization)
data.train_test_split(self.data_split)
model.make_timeseries_dataset(data)
#print(data.dfs)
# Läs in neuralnät
model.load_nn()
model.test()
model.plot_outliers()
model.plot_example()
def threadoperation(start, time, numqueries, timeAccessGenerator, minqueryperiod, maxqueryperiod, periodAccessGenerator, config, logger, x, values):
successfulquerytime = 0
successfulquerycount = 0
failedquerytime = 0
failedquerycount = 0
totalquerytime = 0
totalquerycount = 0
endtime = datetime.now() + timedelta(minutes=runtime)
while True:
if datetime.now() >= endtime:
break
time = datetime.now(timezone('UTC'))
newquerylist = QueryGenerator.generateQueries(start, time, numqueries, timeAccessGenerator, minqueryperiod, maxqueryperiod, periodAccessGenerator);
line = applyOperation(newquerylist[0], config,logger)
#print line[0:10]
if (line[0][0:10] == "Successful"):
print line[1].count
successfulquerytime += float(line[0][12:])
successfulquerycount += 1
totalquerytime += float(line[0][12:])
totalquerycount += 1
elif (line[0][0:6] == "Failed"):
failedquerytime += float(line[0][8:])
failedquerycount += 1
totalquerytime += float(line[0][8:])
totalquerycount += 1
datastructure = [successfulquerytime, successfulquerycount, failedquerytime, failedquerycount, totalquerytime, totalquerycount]
values.put(datastructure)
def printresults():
logger.info('{} {} {} {}'.format(start.strftime("%Y-%m-%d %H:%M:%S"),
end.strftime("%Y-%m-%d %H:%M:%S"),
runtime, queryPerSec))
line = list()
querypermin = queryPerSec * 60
endtime = datetime.now(timezone('UTC')) + timedelta(minutes=runtime)
popularitylist = list()
newquerylist = list()
if filename != "":
newquerylist = QueryGenerator.generateQueriesFromFile(
start, end, querypermin * runtime, timeAccessGenerator,
periodAccessGenerator, filename)
elif isbatch == True:
newquerylist = QueryGenerator.generateQueries(
start, end, querypermin * runtime, timeAccessGenerator,
periodAccessGenerator, popularitylist)
def findUser(self, slackId: str) -> User:
findUserQuery = QueryGenerator.generateFindUser(slackId)
userDoc = self.users.find_one(findUserQuery)
if not userDoc:
return None
else:
foundUser = User(userDoc["_id"], userDoc["name"], userDoc["slackId"], userDoc["lastUseDate"])
return foundUser
def __init__(self, num_clients, duration, mysql_user, mysql_pass,
mysql_host, mysql_port, mysql_socket, database, profile,
tables):
self.num_clients = num_clients
self.duration = duration
self.client_list = []
self.client_stats = []
self.pipe_list = []
self.profile = profile
self.stats_frequency = 0.0
self.total_clients = 0
self.write_sql_to_disk = False
for item in num_clients:
self.total_clients += int(item)
left_over = 0
if str(duration)[-1] != 's':
tduration = int(int(duration) / self.total_clients)
left_over = int(duration) - (tduration * self.total_clients)
duration = tduration
i = 0
for j in range(len(num_clients)):
for n in range(int(self.num_clients[j])):
self.client_stats.append(ClientQueryStats(str(i)))
self.pipe_list.append(multiprocessing.Pipe())
if n == int(self.num_clients[j]) - 1 and left_over > 0:
self.client_list.append(
QueryGenerator(self.pipe_list[i][1], mysql_user,
mysql_pass, mysql_host, mysql_port,
mysql_socket, database, tables,
duration + left_over, profile[j]))
else:
self.client_list.append(
QueryGenerator(self.pipe_list[i][1], mysql_user,
mysql_pass, mysql_host, mysql_port,
mysql_socket, database, tables,
duration, profile[j]))
i += 1
def execute_scheme(self):
model = TimeSeriesPredictionNeuralNet(self.settings)
model.setup()
model.compile_model()
connection = SQLAConnection()
query_generator = QueryGenerator(
self.settings.sensors,
self.settings.start_date,
self.settings.end_date
)
data = Data(query_generator,connection)
if self.args.load_dataframe:
data.load_dfs(date='2020-11-01')
#data.load_extend_dfs(date='2020-11-02')
#data.load_extend_dfs(date='2020-11-03')
#data.load_extend_dfs(date='2020-11-04')
#data.load_extend_dfs(date='2020-11-08')
#data.load_extend_dfs(date='2020-11-12')
#data.load_extend_dfs(date='2020-11-16')
#data.load_extend_dfs(date='2020-11-20')
#data.load_extend_dfs(date='2020-11-24')
#data.load_extend_dfs(date='2020-11-28')
#data.load_extend_dfs(date='2020-12-02')
#data.load_extend_dfs(date='2020-12-18')
#data.load_extend_dfs(date='2020-12-30')
#data.load_extend_dfs(date='2020-12-08')
else:
data.make_df_postgres()
data.find_discontinuities()
data.split_at_discontinuities()
data.preprocess(self.settings.normalization)
data.add_trig()
data.train_test_split(self.data_split)
model.make_timeseries_dataset(data)
model.print_shape()
model.plot_example()
#model.save_dataset()
if self.args.load:
model.load_nn()
model.train()
model.plot_history()
model.evaluate()
model.save_nn(overwrite=True)
model.test()
model.plot_outliers()
model.plot_example()
def execute_scheme(self):
#model = TimeSeriesClassificationNeuralNet(self.settings)
#model = TimeSeriesPredictionNeuralNet(self.settings)
connection = SQLAConnection()
query_generator = QueryGenerator(
self.settings.sensors,
self.settings.start_date,
self.settings.end_date
)
report_generator = ReportGenerator(self.settings)
link_generator = LinkGenerator(self.settings)
data = PostgresData(query_generator,connection)
#data.generate_metadata_report(ReportGenerator(self.settings))
#data.load_df(name=self.settings.dataset_name)
data.make_df()
#data.save_df(name=self.settings.dataset_name)
data.find_discontinuities()
data.split_at_discontinuities()
#data.plot_data()
#data.add_temp()
data.save_dfs(name=self.settings.start_date)
class LegendLinksGetter:
"""
Pobiera wyniki wyszukiwania z różnych przeglądarek.
"""
def __init__(self):
""" Konstruktor... """
self.query_generator = QueryGenerator()
self.response_gatherer = ResponseGatherer()
def search(self, legend_text):
"""
Szuka w Internetach linków do legend.
legend_text jest pełnym tekstem legendy.
Metoda zwraca listę wyników. Wynik w ma pola:
w.url_title - tytuł
w.url - adres url
w.snippet - fragment
w.engine - wyszukiwarka
"""
queries = self.query_generator.generate_queries(legend_text)
results = self.response_gatherer.get_results(queries)
return results
def export(self, query: Query):
qg = QueryGenerator(self.repo)
output_text = qg.generate_query(query.name)
with open(self.file_name, "w") as file_open:
file_open.write(output_text)
# print(graph_conf)
# graph_conf = []
# for i in graph_size:
# for j in range(0, i[2]):
# graph_conf.append([i[0], i[1], query_size])
# Export raw graph from Neo4j DB
ge = GraphExporter()
raw_graph, g1_level, g2_level = ge.export(uri, usename, password,
raw_graph_class_node,
raw_graph_class_edge)
#
# # Create new graph (including dumping to file)
# # gg = GraphGenerator(raw_graph, g1_level, g2_level, graph_size, graph_level, graph_number)
print("Analyze End!")
gg = GraphGenerator(raw_graph, (g1_level + 1), (g2_level + 1), graph_conf,
base_path, base_path_data, base_path_result)
new_graphs, node_id, edge_id = gg.create()
print("BOM Generate End!")
qg = QueryGenerator(new_graphs, graph_conf, node_id, edge_id, base_path,
graph_path, base_path_data, base_path_result)
qg.generate()
print("Query Generate End!")
print(
"[INFO] End. (" +
str(time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))) +
")")
# return
def printresults():
logger.log(
STATS, '{} {} {} {} {}'.format(start.strftime("%Y-%m-%d %H:%M:%S"),
end.strftime("%Y-%m-%d %H:%M:%S"),
runtime, queryPerSec, queryratio))
querypermin = queryPerSec * 60
endtime = datetime.now(timezone('UTC')) + timedelta(minutes=runtime)
line = list()
popularitylist = list()
newquerylist = list()
if filename != "":
newquerylist = QueryGenerator.generateQueriesFromFile(
start, end, querypermin * runtime, timeAccessGenerator,
periodAccessGenerator, querytype, queryratio, filename)
elif isbatch == True:
newquerylist = QueryGenerator.generateQueries(
start, end, querypermin * runtime, timeAccessGenerator,
periodAccessGenerator, popularitylist, querytype, queryratio,
logger)
else:
#logger.info("Run.py start queryendtime "+str(start)+", "+str(endtime))
queryStartInterval = start
queryEndInterval = start + timedelta(minutes=1)
for i in range(0, runtime):
logger.info("Start generating queries for interval " +
str(queryStartInterval) + " - " +
str(queryEndInterval))
newquerylist.extend(
QueryGenerator.generateQueries(
queryStartInterval, queryEndInterval, querypermin,
timeAccessGenerator, periodAccessGenerator,
popularitylist, querytype, queryratio, logger))
queryEndInterval = queryEndInterval + timedelta(minutes=1)
logger.info("Finished generating queries. num queries generated " +
str(len(newquerylist)))
if filename != "" or isbatch == True:
count = 0
time = datetime.now(timezone('UTC'))
logger.info("Time: {}".format(time.strftime("%Y-%m-%d %H:%M:%S")))
nextminute = time + timedelta(minutes=1)
for query in newquerylist:
try:
line.append(
applyOperation(query, config, brokernameurl, logger))
except Exception as inst:
logger.error(type(inst)) # the exception instance
logger.error(inst.args) # arguments stored in .args
logger.error(
inst) # __str__ allows args to be printed directly
x, y = inst.args
logger.error('x =', x)
logger.error('y =', y)
count = count + 1
if count >= querypermin:
timediff = (nextminute -
datetime.now(timezone('UTC'))).total_seconds()
if timediff > 0:
yield gen.sleep(timediff)
count = 0
time = datetime.now(timezone('UTC'))
logger.info("Time: {}".format(
time.strftime("%Y-%m-%d %H:%M:%S")))
nextminute = time + timedelta(minutes=1)
else:
# frequency of queries per millisecond
queryPerMilliSecond = float(queryPerSec) / 1000
# number of samples spaced by 1 millisecond
numSamples = runtime * 60 * 1000
numQueries, querySchedule = genPoissonQuerySchedule(
queryPerMilliSecond, numSamples)
logger.info("Poisson numQueries = " + str(numQueries))
queryScheduleIdx = 0
count = 0
while count < len(newquerylist):
sample = querySchedule[queryScheduleIdx]
#logger.info("Poisson sample is "+str(sample[0])+", "+str(sample[1]))
if (sample[0] == 0):
#logger.info("Sleeping for "+str(sample[1]))
yield gen.sleep(
float(sample[1]) /
1000) # divide by 1000 to convert it into seconds
else:
for i in range(0, sample[0]):
try:
line.append(
applyOperation(newquerylist[count], config,
brokernameurl, logger))
#applyOperation(newquerylist[count], config, brokernameurl, logger)
newquerylist[count].setTxTime(datetime.now())
#logger.info("Running query "+str(sample[0]))
except Exception as inst:
logger.error(type(inst)) # the exception instance
logger.error(
inst.args) # arguments stored in .args
logger.error(
inst
) # __str__ allows args to be printed directly
count = count + 1
if count >= len(newquerylist):
break
queryScheduleIdx = queryScheduleIdx + 1
wait_iterator = gen.WaitIterator(*line)
while not wait_iterator.done():
try:
result = yield wait_iterator.next()
except Exception as e:
logger.error("Error {} from {}".format(
e, wait_iterator.current_future))
def __init__(self):
""" Konstruktor... """
self.query_generator = QueryGenerator()
self.response_gatherer = ResponseGatherer()
def execute_scheme(self):
#model = TimeSeriesClassificationNeuralNet(self.settings)
#model = TimeSeriesPredictionNeuralNet(self.settings)
connection = SQLAConnection()
query_generator = QueryGenerator(
self.settings.sensors,
self.settings.start_date,
self.settings.end_date
)
report_generator = ReportGenerator(self.settings)
link_generator = LinkGenerator(self.settings)
#data = RegularityData(link_generator,connection)
data = AnomalyData(link_generator,connection)
#data.generate_metadata_report(ReportGenerator(self.settings))
#data.make_df()
#data.save_df(name=self.settings.dataset_name)
#data.find_discontinuities()
#data.split_at_discontinuities()
#data.plot_data()
#data.add_temp()
#data.save_dfs(name=self.settings.dataset_name)
#data.load_dfs(date='2020-11-01')
#data.load_extend_dfs(date='2020-11-13')
startdate = datetime.strptime('2020-11-01',config.dateformat)
data.load_dfs(date=datetime.strftime(startdate,config.dateformat))
dates_ahead = 4
mode = 'while'
if mode == 'for':
for i in range(dates_ahead):
data.load_extend_dfs(date=datetime.strftime(startdate+timedelta(days=i), config.dateformat))
elif mode == 'while':
tdate = startdate
while tdate.date() != date.today():
try:
data.load_extend_dfs(date=datetime.strftime(tdate, config.dateformat))
except FileNotFoundError:
pass
tdate = tdate+timedelta(days=1)
data.purge_empty_dfs()
data.preprocess()
data.merge_dfs()
#data.plot_data()
#data.find_correlation()
anomaly_settings = AnomalySettings()
kmeans_settings = KMeansSettings()
start_hour = '00:00:00'
end_hour = '23:59:59'
data.filter_hours(start_hour,end_hour)
data.purge_empty_time_filtered_dfs()
#data.plot_filtered_hours(plot_objects=False)
data.set_object_settings(anomaly_settings)
anomaly_name = f"{startdate}_{mode}_{start_hour}_{end_hour}_{anomaly_settings.anomaly_sensor}_anomaly"
print(os.listdir(config.anomaly_path))
print(anomaly_name)
if f"{anomaly_name}.json" in os.listdir(config.anomaly_path):
data.load_objects(name=f"{anomaly_name}.json")
print(f"{anomaly_name} loaded")
else:
for feature in anomaly_settings.anomaly_sensor:
#data.locate_anomalies_filtered_dfs(feature)
data.locate_objects_dfs(feature)
#data.save_plots(feature)
#data.plot_filtered_hours(foi = feature)
data.save_objects(name=anomaly_name)
kmeans = KMeansClustering(data.objects,kmeans_settings)
kmeans.fit_Kmeans()
#sensor_prediction = SensorPrediction(data.anomalies,self.settings)
data.plot_filtered_hours(foi = 'acc1_ch_x')#,project_anomalies = 'acc1_ch_z')
pca = PCAAnomalies(data.objects,self.settings)
pca.fit_PCA()
pca.save_pca(f'{anomaly_name}_pca')
pca.set_labels(kmeans.send_labels())
#pca.get_cov()
#anomaly_key, df_number = pca.get_argmax(col='sigma')
#data.plot_regularities()
pca.plot_components_labels(n_categories = kmeans_settings.n_clusters)
pca.scree_plot()
pca.plot_hist_pca()
#pca.plot_components_3d()
pca.plot_components(features = ['Duration','frequency'])
#### LOAD Phase ####
print "Pre loading segments and adding to deep storage"
segmentlist = RealTimeNode.generateSegments(preloadsegment)
deepstorage.extend(segmentlist)
segmentrunningcount = len(deepstorage)
RealTimeNode.printlist(segmentlist)
#### RUN Phase ####
for time in xrange(totaltime):
#Generating Queries
print "Generating Queries"
maxquerysize = min(segmentrunningcount, querymaxsize)
minquerysize = min(queryminsize, maxquerysize)
newquerylist = QueryGenerator.generateQueries(
queryperinterval, segmentrunningcount,
DistributionFactory.createSegmentDistribution(
querysegmentdistribution), minquerysize, maxquerysize,
DistributionFactory.createSizeDistribution(querysizedistribution))
Utils.printQueryList(newquerylist)
allquerylist.extend(newquerylist)
#Routing Queries
for strategy in dynamicstrategies:
strategy.routeQueries(newquerylist, segmentrunningcount, time)
if time % segmentinterval == 0:
#Generating Segments indexed starting from 1
print "Generating Segments and adding to deep storage"
newsegments = RealTimeNode.generateSegments(segmentsperinterval)
RealTimeNode.printlist(newsegments)
segmentlist.extend(newsegments)
def addList(self, userId: str, listName: str) -> None:
addListQuery = QueryGenerator.generateAddList(userId, listName)
self.lists.insert_one(addListQuery)
def addItemToList(self, item: str, list: str) -> None:
addItemQuery = QueryGenerator.generateAddItemToList(item, list)
self.listItems.insert_one(addItemQuery)
def execute_scheme(self):
#model = TimeSeriesClassificationNeuralNet(self.settings)
model = TimeSeriesPredictionNeuralNet(self.settings)
connection = SQLAConnection()
query_generator = QueryGenerator(
self.settings.sensors,
self.settings.start_date,
self.settings.end_date
)
report_generator = ReportGenerator(self.settings)
link_generator = LinkGenerator(self.settings)
data = AnomalyData(link_generator,connection)
#data.generate_metadata_report(ReportGenerator(self.settings))
#data.make_df()
#data.save_df(name=self.settings.dataset_name)
#data.find_discontinuities()
#data.split_at_discontinuities()
#data.plot_data()
#data.add_temp()
#data.save_dfs(name=self.settings.dataset_name)
#data.load_dfs(date='2020-11-01')
#data.load_extend_dfs(date='2020-11-13')
startdate = datetime.strptime('2020-11-20',config.dateformat)
data.load_dfs(date=datetime.strftime(startdate,config.dateformat))
dates_ahead = 2
mode = 'while'
if mode == 'for':
for i in range(dates_ahead):
data.load_extend_dfs(date=datetime.strftime(startdate+timedelta(days=i), config.dateformat))
elif mode == 'while':
tdate = startdate
while tdate.date() != date.today():
try:
data.load_extend_dfs(date=datetime.strftime(tdate, config.dateformat))
except FileNotFoundError:
pass
tdate = tdate+timedelta(days=1)
data.purge_empty_dfs()
data.preprocess()
data.merge_dfs()
#data.find_correlation()
anomaly_settings = AnomalySettings()
data.filter_hours('02:00:00','05:00:00')
data.set_anomaly_settings(anomaly_settings)
for feature in anomaly_settings.anomaly_sensor:
#data.locate_anomalies_filtered_dfs(feature)
data.locate_anomalies_dfs(feature)
#data.save_plots(feature)
#data.plot_filtered_hours(foi = feature)
#data.plot_filtered_hours(foi = 'acc1_ch_z')#,project_anomalies = 'acc1_ch_z')
pca = PCAAnomalies(data.anomalies)
pca.fit_PCA()
#pca.get_cov()
#anomaly_key, df_number = pca.get_argmax(col='sigma')
pca.plot_components()
pca.scree_plot()
pca.plot_hist()
pca.plot_components_3d()
if time >= warmuptime:
#Generating Queries
print "Generating Queries"
if changequerydistribution == True and time == changedistributionat:
print "Distribution Change"
querysegmentdistribution=newquerysegmentdistribution
maxqueryperiod = min(time, querymaxsize)
minqueryperiod = min(queryminsize, maxqueryperiod)
numqueries = queryperinterval;
if burstyquery == True and time % burstyqueryinterval == 0:
print "Query Burst"
numqueries *= burstyquerymultiplier
newquerylist = QueryGenerator.generateQueries(time, numqueries, deepstorage, DistributionFactory.createSegmentDistribution(querysegmentdistribution), minqueryperiod, maxqueryperiod, DistributionFactory.createSizeDistribution(querysizedistribution));
Utils.printQueryList(newquerylist)
allquerylist.extend(newquerylist)
#Routing Queries
for strategy in dynamicstrategies:
strategy.routeQueries(newquerylist, segmentrunningcount, time)
#Placing Segments
if time % coordinatorinterval == 0:
for strategy in dynamicstrategies:
strategy.placeSegments(segmentlist, time, config)
segmentlist = []
#Print Statistics
for strategy in dynamicstrategies:
print("Segments per Interval: %s" % segmentsperinterval)
#### LOAD Phase ####
print "Pre loading segments and adding to deep storage"
segmentlist = RealTimeNode.generateSegments(preloadsegment)
deepstorage.extend(segmentlist)
segmentrunningcount = len(deepstorage)
RealTimeNode.printlist(segmentlist)
#### RUN Phase ####
for time in xrange(totaltime):
#Generating Queries
print "Generating Queries"
maxquerysize = min(segmentrunningcount, querymaxsize)
minquerysize = min(queryminsize, maxquerysize)
newquerylist = QueryGenerator.generateQueries(queryperinterval, segmentrunningcount, DistributionFactory.createSegmentDistribution(querysegmentdistribution), minquerysize, maxquerysize, DistributionFactory.createSizeDistribution(querysizedistribution));
Utils.printQueryList(newquerylist)
allquerylist.extend(newquerylist)
#Routing Queries
for strategy in dynamicstrategies:
strategy.routeQueries(newquerylist, segmentrunningcount, time)
if time % segmentinterval == 0:
#Generating Segments indexed starting from 1
print "Generating Segments and adding to deep storage"
newsegments = RealTimeNode.generateSegments(segmentsperinterval)
RealTimeNode.printlist(newsegments)
segmentlist.extend(newsegments)
deepstorage.extend(newsegments)
segmentrunningcount = len(deepstorage)
def export(self, query: Query):
qg = QueryGenerator(self.repo)
output_text = qg.generate_query(query.name)
with open(self.file_name, "w") as file_open:
file_open.write(output_text)
querysizedistribution = config.getQuerySizeDistribution() queryminsize = config.getQueryMinSize() querymaxsize = config.getQueryMaxSize() historicalnodecount = config.getHistoricalNodeCount() placementstrategy = config.getPlacementStrategy() replicationfactor = config.getReplicationFactor() percentreplicate = config.getPercentReplicate() #Generating Segments indexed starting from 1 print "Generating Segments" segmentlist = RealTimeNode.generateSegments(segmentcount) RealTimeNode.printlist(segmentlist) #Generating Queries print "Generating Queries" querylist = QueryGenerator.generateQueries(querycount, segmentcount, DistributionFactory.createSegmentDistribution(querysegmentdistribution), queryminsize, querymaxsize, DistributionFactory.createSizeDistribution(querysizedistribution)); printQueryList(querylist) ### DRUID COST BASED placementstrategy = "druidcostbased" for replicationfactor in xrange(2, 3): ### Random Routing Stretagy routingstrategy = "random" runExperiment(historicalnodecount, segmentlist, percentreplicate, replicationfactor, querylist, placementstrategy, routingstrategy) ### Connection Count Based Strategy routingstrategy = "chooseleastloaded" runExperiment(historicalnodecount, segmentlist, percentreplicate, replicationfactor, querylist, placementstrategy, routingstrategy) ### RANDOM BALL BASED
self.__print_line__(str(line))
sqlslap_process.wait()
def run_scenarios(self,
FLAGIN_FLAGOUT=False,
no_backend=False,
without_query_rules=False,
distribution='normal'):
self.__apply_query_rules_to_proxy_sql__(FLAGIN_FLAGOUT, no_backend,
without_query_rules)
self.__load_tests__(distribution)
if __name__ == "__main__":
query_generator = QueryGenerator()
query_rules_generator = QueryRulesGenerator()
benchmark = Benchmark(query_generator, query_rules_generator)
print('----------------------Normal Distribution-------------------------')
print('query rules without flagIN and flagOUT and with backend server')
benchmark.run_scenarios(FLAGIN_FLAGOUT=False,
no_backend=False,
distribution='normal')
print('******************************************************************')
print('query rules with flagIN and flagOUT and with backend server')
benchmark.run_scenarios(FLAGIN_FLAGOUT=True,
no_backend=False,
distribution='normal')
print('******************************************************************')
print('query rules without flagIN and flagOUT and without backend server')
def threadoperation(start, time, numqueries, timeAccessGenerator,
minqueryperiod, maxqueryperiod, periodAccessGenerator,
config, logger, x, values):
successfulquerytime = 0
successfulquerycount = 0
failedquerytime = 0
failedquerycount = 0
totalquerytime = 0
totalquerycount = 0
endtime = datetime.now() + dt.timedelta(minutes=runtime)
currentSegRank = []
#oldest_timestamp = start.total_seconds()
break_flag = 0
#print "16"
while True:
#while break_flag <1:
break_flag = break_flag + 1
#print "1"
if datetime.now() >= endtime:
break
time = datetime.now(timezone('UTC'))
#newquerylist = QueryGenerator.generateQueries(start, time, numqueries, timeAccessGenerator, minqueryperiod, maxqueryperiod, periodAccessGenerator);
if (len(currentSegRank) == 0):
#print "4"
y = time - start
z = y.total_seconds()
x = dt.timedelta(seconds=z)
distance = x.total_seconds()
for i in range(0, int(round(distance))):
timepoint = start + dt.timedelta(0, i)
currentSegRank.append(timepoint)
else:
#y = time.total_seconds()
#z = start.total_seconds()
#print "2"
new_interval = (time - start).total_seconds()
#print "time"
print time
#print "start"
print start
print new_interval
for i in range(0, int(round(new_interval))):
samples = randZipf(1 + len(currentSegRank), 1.2, 1)
timepoint = start + dt.timedelta(0, i)
print "add to rank: index"
print samples[0]
print "timepoint"
print timepoint
print "list seg size %d " % len(currentSegRank)
#len(currentSegRank)
currentSegRank.insert(samples[0], timepoint)
print currentSegRank
print filename
if (filename != ""):
#break_flag = 1
newquerylist = QueryGenerator.generateQueriesFromFile(
start, time, timeAccessGenerator, minqueryperiod,
maxqueryperiod, periodAccessGenerator, filename)
#elif(accessdistribution == "dynamiczip"):
#newquerylist = QueryGenerator.generateQueries(start, time, numqueries, timeAccessGenerator, minqueryperiod, maxqueryperiod, periodAccessGenerator, currentSegRank)
else:
#print "3"
newquerylist = QueryGenerator.generateQueries(
start, time, numqueries, timeAccessGenerator, minqueryperiod,
maxqueryperiod, periodAccessGenerator, currentSegRank)
line = applyOperation(newquerylist[0], config, logger)
print line[0:10]
if (line[0][0:10] == "Successful"):
print line[1].count
successfulquerytime += float(line[0][12:])
successfulquerycount += 1
totalquerytime += float(line[0][12:])
totalquerycount += 1
elif (line[0][0:6] == "Failed"):
failedquerytime += float(line[0][8:])
failedquerycount += 1
totalquerytime += float(line[0][8:])
totalquerycount += 1
datastructure = [
successfulquerytime, successfulquerycount, failedquerytime,
failedquerycount, totalquerytime, totalquerycount
]
values.put(datastructure)
def main(usename, password, raw_graph_class_node, raw_graph_class_edge,
query_size, graph_size, socket, uri, graph_path):
print(
"[INFO] Start. (" +
str(time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))) +
")")
# Neo4j database configuration
# uri = "bolt://127.0.0.1:7687"
# usename = "neo4j"
# password = "******"
# raw_graph_class_node = "item_1"
# raw_graph_class_edge = "rel_1"
# 使用datetime.now()
now = datetime.datetime.now()
timestr = str(now.year) + str(now.month) + str(now.day) + str(
now.hour) + str(now.minute) + str(now.second)
# 需要根据项目所在目录进行改变的
#获取项目的路径
base_path = os.getcwd()
base_path_data = os.getcwd() + "/data/data_" + timestr
base_path_result = os.getcwd() + "/result/result_" + timestr
print("base_path" + base_path)
if not os.path.exists(base_path_data):
os.makedirs(base_path_data)
if not os.path.exists(base_path_result):
os.makedirs(base_path_result)
#获取服务器上neo4j（图数据库）的路径
# graph_path = "/Users/mark/Documents/MasterLife/GraphDB/neo4j-community-3.4.6"
# New graph configuration: graph_conf[size, level, query_num]
# query_size = 20
# graph_size = [ [2000, 5, 1] ]
# query_size = 1
# graph_size = [ [3000, 7, 1], [6000, 10, 1], [12000, 14, 1], [24000, 20, 1] ]
# 2000，5，1&2000, 6, 1
# graph_size = "2000,5,1"
_graph_size = graph_size.split("&")
graph_conf = []
for i in _graph_size:
_i = i.split(",")
for j in range(0, int(_i[2])):
graph_conf.append([int(_i[0]), int(_i[1]), query_size])
# print(graph_conf)
# graph_conf = []
# for i in graph_size:
# for j in range(0, i[2]):
# graph_conf.append([i[0], i[1], query_size])
# Export raw graph from Neo4j DB
socket.wfile.write(("Start to Analyze ！<br>").encode())
ge = GraphExporter()
raw_graph, g1_level, g2_level = ge.export(uri, usename, password,
raw_graph_class_node,
raw_graph_class_edge)
#
# # Create new graph (including dumping to file)
# # gg = GraphGenerator(raw_graph, g1_level, g2_level, graph_size, graph_level, graph_number)
print("Analyze End!")
socket.wfile.write(("Analyze End!！<br>").encode())
gg = GraphGenerator(raw_graph, (g1_level + 1), (g2_level + 1), graph_conf,
base_path, base_path_data, base_path_result)
new_graphs, node_id, edge_id = gg.create()
print("BOM Generate End!")
socket.wfile.write(("BOM Generate End!<br>").encode())
qg = QueryGenerator(new_graphs, graph_conf, node_id, edge_id, base_path,
graph_path, base_path_data, base_path_result)
qg.generate()
print("Query Generate End!")
socket.wfile.write(("Query Generate End!<br>").encode())
print(
"[INFO] End. (" +
str(time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))) +
")")
return
=======
# Instansiera modell
>>>>>>> docker-test
model = TimeSeriesPredictionNeuralNet(self.settings)
# Importera modellinställningar
model.setup()
<<<<<<< HEAD
model.compile_model()
if self.args.load_dataset:
model.load_dataset()
else:
connection = SQLAConnection()
query_generator = QueryGenerator(
self.settings.sensors,
self.settings.start_date,
self.settings.end_date
)
data = Data(query_generator,connection)
if self.args.load_dataframe:
data.load_dfs(date='2020-11-01')
else:
data.make_df_postgres()
data.find_discontinuities()
data.split_at_discontinuities()
data.preprocess(self.settings.normalization)
#data.fast_fourier_transform()
#data.wawelet()
#data.STL()
data.add_trig()
#data.add_temp()
