def nn_analysis(self):
if not os.path.isfile(
self.file.split("/healthy")[0] + "/manual/" +
self.file.split(".")[0][-4:] + '.tif'):
print(
"There's no original mask to compare! Analysis cannot be performed."
)
return
self.nn_analysis = Analysis()
epochs = [25, 50, 75, 100]
squared_errors = []
for epoch in epochs:
self.epochs = epoch
self.nn_train()
self.nn_predict()
squared_error = self.nn_analysis.mean_squared_error(
self.predicted_original, self.predicted)
squared_errors.append([epoch, squared_error])
print(squared_errors)
file = open('squared_errors.txt', 'a')
file.write(self.file)
for error in squared_errors:
file.write("epoch: " + str(error[0]) + "\t error: " +
str(error[1]) + "\n")
file.write("\n")
file.close()
def test_iter(self):
x = []
y = []
self.var_iter_checkbox.set(True)
self.pts_transformation.alpha = 2
self.pts_transformation.detectors_amount = 99
self.pts_transformation.width = 180 * 2
self.filter_props.gamma = 2.4
self.filter_props.gauss = 1.0
self.pts_transformation.generate_all_positions(self.input_picture)
is_end = False
i = 0
while not is_end:
self.sinogram, is_end = self.pts_transformation.make_sinogram_iter(
self.input_picture)
self.display_picture(Image.fromarray(self.sinogram), 'sinogram')
self.restored_picture = self.pts_transformation.restore_picture(
self.sinogram, len(self.input_picture), self.filter_props)
self.display_picture(Image.fromarray(self.restored_picture),
'output')
self.root.update_idletasks()
mse = Analysis.mean_squared_error(self.input_picture,
self.restored_picture)
x.append(i)
y.append(mse)
i += 1
print("Iter =", i, "error =", mse)
Analysis.draw_plot(x, y, "Iteracja", "Błąd średniokwadratowy", "iter")
def run_truss_design2():
A = Analysis()
c = np.array([[1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0],
[0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0],
[0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0],
[0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, 1, 0, 0],
[0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0]])
sx = np.array([[1, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0],
[0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0]])
sy = np.array([[0, 1, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 1],
[0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0]])
x = np.array([0, 6.5, 13, 20.5, 28, 0, 6.5, 13, 20.5, 28])
y = np.array([0, 0, 0, 0, 0, 6.5, 6.5, 6.5, 6.5, 6.5])
x = x.reshape(1, 10)
y = y.reshape(1, 10)
l = np.array(
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -50, 0, 0, 0, 0, 0, 0, 0])
l = l.reshape(20, 1)
A.set_truss_design(c, sx, sy, x, y, l)
A.construct_A_mat()
A.print_output()
A.print_formatted_per_mem()
def neural_network(self):
"""This function runs the neural network process for the displayed image"""
try:
if self.image_object.file_type == ImageType.JPG or self.image_object.file_type == ImageType.PNG:
predict = Net.testImage(self.image_object.get_file_path())
if predict == 'normal':
prediction = 'Normal'
else:
prediction = 'COVID-19'
if self.analysis is None:
self.analysis = Analysis(slices_number=self.image_object.total_slice_number)
self.add_result_to_analysis_neural_network(prediction,
self.image_object.get_current_slice_number_to_show())
logging.debug('Added neural network result to reports: ')
else:
prediction = "Network accepts only jpg or png files!"
popup = Popup(title='Result',
content=Label(text=prediction),
size=(400, 400),
size_hint=(None, None))
popup.open()
except Exception as error:
logging.error("Neural network classification: " + str(error))
self._error_popup = ErrorPopup(message=str(error))
def train_evaluate(search_params):
hyperparameters = {}
pick_kwargs = {}
for k in list(search_params.keys()):
if k in ['w_dfh', 'w_sharpe', 'w_100d', 'v_100d', 'v_dfh', 'v_rfl']:
pick_kwargs[k] = search_params[k]
else:
hyperparameters[k] = search_params[k]
hyperparameters['pick_kwargs'] = pick_kwargs
print('------------')
print(json.dumps(hyperparameters, indent=2, sort_keys=True))
sim = Sim(neptune=neptune,
period='2y',
timedelay=100,
window=100,
timestep=1,
budget=5000,
stockPicks=5,
avoidDowntrends=True,
sellAllOnCrash=False,
**hyperparameters)
stats = sim.run()
analysis = Analysis(neptune=neptune,
stats=stats,
positions=sim.portfolio.holdings,
prices=sim.downloader.prices)
#analysis.chart()
output, advanced_stats, obj_stats = analysis.positionStats()
for k in list(obj_stats.keys()):
neptune.log_metric(k, obj_stats[k])
print(output)
#neptune.log_artifact('data/output_1y.pkl')
sharpe = analysis.sharpe()
stats = sim.portfolio.summary()
if math.isnan(sharpe) or math.isinf(sharpe) or sharpe <= -2 or sharpe >= 5:
sharpe = -5
#neptune.log_metric('sharpe', sharpe)
#neptune.log_metric('start_value', 5000)
#neptune.log_metric('end_value', stats['total_value'])
report = {
'hyperparameters': hyperparameters,
'sharpe': sharpe,
'end_value': stats['total_value'],
'gains': (stats['total_value'] - 5000.0) / 5000.0
}
neptune.log_text('report', json.dumps(report, indent=2, sort_keys=True))
return sharpe
def nn_confusion(self):
self.nn_analysis = Analysis()
path = self.file.split(".")
name = path[0].split("/")[-1]
original = io.imread(Sample.output_path + name +
Sample.output_extension,
as_grey=True)
self.nn_analysis.confusion(self.predicted, original)
def on_analysis():
dateFrom = _getParam('from', '2011')
dateTo = _getParam('to', '2018')
ccy = _getParam('ccy', 'GBP')
analysis = Analysis(dateFrom, dateTo, ccy)
results = analysis.YearlySpend()
yearTotals = analysis.yearTotals()
return (render_template('analysis.html', yearly_spend = results['salary'], reimbursements=results['reimbursements'], expenses=results['expenses'], withholding=results['withholding'], year_totals=yearTotals, date_from=dateFrom, date_to=dateTo, date_range = analysis.DateRange(), ccy=ccy))
def analyze():
b = Budget()
b.add(i, t, d, e)
b.report()
m = Metrics(i, b, e, d, a)
m.calculate()
m.report()
ana = Analysis(p, i, d, b, e, a, m)
ana.do_all()
def __init__(self, *args, **kwargs):
super(RootWidget, self).__init__(*args, **kwargs)
try:
self.image_object = CTJpgImage(GUI_FOLDER, START_IMAGE)
self.examination_type = ExaminationType.CT
# new analysis initialization
self.analysis = Analysis(slices_number=self.image_object.total_slice_number)
except Exception as error:
logging.critical("Init root widget: " + str(error))
self._error_popup = ErrorPopup(message=str(error))
def __init__(self):
Analysis.__init__(self)
self.data = pd.read_sql(session.query(covidData).__str__(), con=engine)
self.data.columns = [
'SNo', 'ObservationDate', 'Province/State', 'Country/Region',
'Last Update', 'Confirmed', 'Deaths', 'Recovered'
]
self.plotOptions = [
"line", "bar", "barh", "hist", "box", "density", "area", "scatter"
]
def main(argv):
if len(argv) == 2:
attackType = argv[1].strip()
else:
attackType = "sql"
sa = Analysis(7474)
query = sa.prepareQueryStatic(attackType)
print('the query is ')
print(query)
result, elapsed_time = sa.runTimedQuery(query)
print(result)
def test_simple_crow_score_regression(self):
data_set = self.dataImporter.import_all_data(
self.test_photos_csv_file_path, self.photos_path)
results = Learning.simple_crow_score_regression(data_set,
epochs=1,
batch_size=2)
Analysis.store_raw_result(self.result_path, results)
Analysis.process_result(self.result_path)
def use_Analysis(n, k):
analysis = Analysis()
start = time.time()
all_solutions = analysis.get_analysis(n, k)
end = time.time()
average = (end - start) / (k**n)
print("k = " + str(k))
for (key, value) in all_solutions:
print(str(key) + ": " + str(value) + "%")
print("total time: " + str(end - start))
print("average time: " + str(average))
analysis.visualize(all_solutions, k)
def main(argv):
attackType = argv[1].strip()
sa = Analysis(7474)
query = sa.prepareQueryStatic(attackType)
print('the query is ')
print(query)
result, elapsed_time = sa.runTimedQuery(query)
print(result)
writeToFile(result, elapsed_time, attackType)
result, elapsed_time = sa.runTimedQuery("g.V().includeMap()")
writeIncludeMapToFile(result, elapsed_time)
def populate_stats_table(self):
self.experimentStatsTable.setRowCount(len(self.experiment.animal_list.keys()))
for m, mouse in enumerate(sorted(list(self.experiment.animal_list.keys()))):
this_mouse = self.experiment.animal_list[mouse]
id = QtWidgets.QTableWidgetItem(this_mouse.id)
total_trials = QtWidgets.QTableWidgetItem(str(Analysis.n_trials_performed(this_mouse)))
trials_last24h = QtWidgets.QTableWidgetItem(str(Analysis.n_trials_last_24(this_mouse)))
self.experimentStatsTable.setItem(m, 0, id)
self.experimentStatsTable.setItem(m, 1, total_trials)
self.experimentStatsTable.setItem(m, 2, trials_last24h)
class ReportWriter(object):
def __init__(self, config, logger):
self.analysis = Analysis(config.get_value("SHAREABLE", "trade_log_file_name"))
self.outfn = config.get_value("SHAREABLE", "stat_file_name")
self.logger = logger
def write(self):
report_text = ""
max_mo_length = len(str(self.analysis.max_markout_period()))
max_ticker_length = max([len(x) for x in self.analysis.get_tickers()]) + 2
out_cols = ["Markout Period", "Ticker", "Trade Count", "Return mu", "Return tstat", "Return sharpe"]
header = ["{item:>{item_width}}".format(item=col, item_width=len(col)) for col in out_cols]
report_text += "\t".join(header) + "\n"
for mo_period in self.analysis.get_mo_periods():
for ticker in self.analysis.get_tickers():
out = {
'Markout Period' : mo_period,
'Ticker' : ticker,
'Trade Count' : self.analysis.count(mo_period, ticker),
'Return mu' : "{:0.4f}".format(self.analysis.mean(mo_period, ticker)),
'Return tstat' : "{:0.2f}".format(self.analysis.tstat(mo_period, ticker)),
'Return sharpe' : "{:0.2f}".format(self.analysis.sharpe(mo_period, ticker))
}
output = []
for column in out_cols:
if column in out:
output.append( "{item:>{item_width}}".format(item=out[column], item_width=len(column)) )
report_text += "\t".join(output) + "\n"
report_text += "\n"
with open(self.outfn, 'w') as f:
f.write(report_text)
self.logger.info("Report written to " + self.outfn)
def display_group_performance(self):
n_longest = 0
all_performance = list()
bin_size = int(self.binSizeSpin.value())
for animal_id in self.parent.experiment.animal_list.keys():
animal = self.parent.experiment.animal_list[animal_id]
if animal_id != 'default':
this_performance = Analysis.binned_performance(animal, bin_size)
all_performance.append(this_performance)
if len(this_performance) > n_longest:
n_longest = len(this_performance)
performance_matrix = np.empty((len(self.parent.experiment.animal_list)-1, n_longest))
performance_matrix[:] = np.nan
for p, perf in enumerate(all_performance):
performance_matrix[p][0:len(perf)] = perf
av_performance = np.nanmean(performance_matrix, 0)
std_performance = np.nanstd(performance_matrix, 0)
self.groupPerformanceView.plotItem.clear()
self.groupPerformanceView.plotItem.plot(av_performance)
self.groupPerformanceView.plotItem.plot(av_performance + std_performance, pen='m')
self.groupPerformanceView.plotItem.plot(av_performance - std_performance, pen='m')
# Guide lines
self.groupPerformanceView.plotItem.plot(np.ones(len(av_performance)) * 0.5, pen='r')
self.groupPerformanceView.plotItem.plot(np.ones(len(av_performance)) * 0.8, pen='g')
self.groupPerformanceView.setYRange(-0.1, 1.1)
def __init__(self):
"""
Initializes a new SMACrossOverDelayed object.
@rtype: None
"""
# Note: Each candlestick represents 1 minute
self.crossOverDurationShorter = 15 # SMA duration 1 (SHORTER)
self.crossOverDurationLonger = 50 # SMA duration 2 (LONGER)
self.baseLongPosition = 600 # base long position size
self.baseShortPosition = 600 # base short position size
self.crossOverDelayForLongTrades = 3 # number of crossover ticks before executing a long position
self.crossOverDelayForShortTrades = 3 # number of crossover ticks before executing a long position
self.analyzer = Analysis.getInstance(
) # get the singleton instance of Analysis
self.data = Data.getInstance() # get the singleton instance of Data
self.smaShorter = None # shorter-term SMA object
self.smaShorterList = None # list of shorter-term SMA values
self.smaLonger = None # longer-term SMA object
self.smaLongerList = None # list of longer-term SMA values
self.strategyName = "SMA Crossover(" + str(self.crossOverDurationShorter) + "," + \
str(self.crossOverDurationLonger) + ")" + " D=" + \
str(self.crossOverDelayForLongTrades)
def graph():
if request.method == 'GET':
return render_template('Graph.html')
elif request.method == 'POST':
x = int(request.form['graph'])
Analysis(x) #.selection(x)
time.sleep(5)
return render_template('Images.html')
def test_detectors(self):
detectors = np.arange(3, 102, 4)
x = []
y = []
self.pts_transformation.alpha = 2
self.pts_transformation.width = 180 * 2
for i in detectors:
self.pts_transformation.detectors_amount = i
self.refresh()
mse = Analysis.mean_squared_error(self.input_picture,
self.restored_picture)
print("Detectors =", i, "error =", mse)
self.root.update_idletasks()
x.append(i)
y.append(mse)
Analysis.draw_plot(x, y, "Liczba detektorów", "Błąd średniokwadratowy",
"detectors")
def handle(self):
data = self.request.recv(BUF_SIZE)
data = data.decode()
print(data)
analysis_data = Analysis().analysis(data)
print(str(analysis_data).encode())
self.request.send(str(analysis_data).encode())
return
def test_width(self):
widths = np.arange(5, 181, 5)
x = []
y = []
self.pts_transformation.alpha = 2
self.pts_transformation.detectors_amount = 99
for i in widths:
self.pts_transformation.width = i * 2
self.refresh()
mse = Analysis.mean_squared_error(self.input_picture,
self.restored_picture)
print("Width =", i, "error =", mse)
self.root.update_idletasks()
x.append(i)
y.append(mse)
Analysis.draw_plot(x, y, "Kąt rozwarcia stożka [°]",
"Błąd średniokwadratowy", "width")
def test_alpha(self):
alphas = np.arange(2, 91, 2)
x = []
y = []
self.pts_transformation.detectors_amount = 99
self.pts_transformation.width = 180 * 2
for i in alphas:
self.pts_transformation.alpha = i
self.refresh()
mse = Analysis.mean_squared_error(self.input_picture,
self.restored_picture)
print("Alpha =", i, "error =", mse)
self.root.update_idletasks()
x.append(i)
y.append(mse)
Analysis.draw_plot(x, y, "Kąt α [°]", "Błąd średniokwadratowy",
"alpha")
def run_analysis(self, conn):
# Analyse each unanalysed object
for unanalysed_object in self.unanalysed_shapes:
# Start analysis on the new image object
analysis = Analysis(unanalysed_object["image"])
# Run each analysis
shape = analysis.get_shape()
shape_colour = analysis.get_shape_colour()
character = analysis.get_character()
character_colour = analysis.get_character_colour()
character_orientation = analysis.get_character_orientation()
# Update occurances dictionaries
self.add_shape(shape)
self.add_shape_colour(shape_colour)
self.add_character(character)
self.add_character_colour(character_colour)
self.add_character_orientation(character_orientation)
# Increment the number of analysed objects
self.num_objects += 1
# Reset unanalysed objects list
self.unanalysed_shapes = []
# Check if object can be sent to ground
if (self.completed):
self.send_object_to_ground()
# Send analysed ODLC class back to the manager process
conn.send(self)
def get_end_results(self):
# get optimized pulse and propagation
# get and save inter vects
self.anly = Analysis(self.sys_para, self.tfs.final_state,
self.tfs.ops_weight, self.tfs.unitary_scale,
self.tfs.inter_vecs)
self.save_data()
self.display()
if not self.show_plots:
self.conv.save_evol(self.anly)
self.uks = self.Get_uks()
if not self.sys_para.state_transfer:
self.Uf = self.anly.get_final_state()
else:
self.Uf = []
def test_gamma(self):
gammas = np.arange(0.2, 4.1, 0.2)
x = []
y = []
self.pts_transformation.alpha = 2
self.pts_transformation.detectors_amount = 99
self.pts_transformation.width = 180 * 2
for i in gammas:
self.filter_props.gamma = i
self.refresh()
mse = Analysis.mean_squared_error(self.input_picture,
self.restored_picture)
print("Gamma =", i, "error =", mse)
self.root.update_idletasks()
x.append(i)
y.append(mse)
Analysis.draw_plot(x, y, "Wartość γ", "Błąd średniokwadratowy",
"gamma")
def test_gauss(self):
gauss = np.arange(0, 3.1, 0.1)
x = []
y = []
self.pts_transformation.alpha = 2
self.pts_transformation.detectors_amount = 99
self.pts_transformation.width = 180 * 2
self.filter_props.gamma = 2.4
for i in gauss:
self.filter_props.gauss = i
self.refresh()
mse = Analysis.mean_squared_error(self.input_picture,
self.restored_picture)
print("Gauss =", i, "error =", mse)
self.root.update_idletasks()
x.append(i)
y.append(mse)
Analysis.draw_plot(x, y, "Odchylenie standardowe",
"Błąd średniokwadratowy", "gauss")
def add_Menu(self):
dd = self.text_doc.textCursor().selectedText() # cursor выделенное
start_elem = self.text_doc.textCursor().selectionStart(
) # the number of the first element of the selected text
end_elem = self.text_doc.textCursor().selectionEnd(
) # the number of the last element of the selected text
self.dialog = NewGroup_Menu(dd, self.data, self.bmks_filename,
start_elem, end_elem, self)
self.dialog.show()
def generate_result(self, target_function, params):
result_path = self.make_result_file_path(params)
tensorboard_path = self.make_tensorboard_path(params)
os.makedirs(result_path, exist_ok=True)
config = self.parameter_space.apply_instantiation(params)
performance, raw_analysis = target_function({
**config, "result_path":
result_path,
"tensorboard_path":
tensorboard_path
})
Analysis.store_raw_result(result_path, raw_analysis)
return performance
def onProfileClicked(self):
"""
Sets the strategy used in the simulator to the strategy with this
profile's strategyIndex.
@rtype: None
"""
analyzer = Analysis.getInstance()
analyzer.strategy = self.strategyIndex
data = Data.getInstance()
data.setDataTimerToReset()
def main():
A = Analysis()
c = np.array([[1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0],
[0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0],
[0, 0, 1, 1, 0, 0, 0, 1, 0, 1, 0, 0, 0],
[0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1],
[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1]])
sx = np.array([[0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0],
[0, 0, 0], [0, 0, 0], [1, 0, 0]])
sy = np.array([[0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 1],
[0, 0, 0], [0, 0, 0], [0, 1, 0]])
x = np.array([0, 4, 8, 12, 12, 8, 4, 0])
y = np.array([4, 8, 8, 4, 0, 4, 4, 0])
x = x.reshape(1, 8)
y = y.reshape(1, 8)
l = np.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -25, 0])
l = l.reshape(16, 1)
A.set_truss_design(c, sx, sy, x, y, l)
# A.mat_load()
# print(A.get_x_vect())
# print(A.get_y_vect())
A.construct_A_mat()
class ReportWriter(object):
def __init__(self, config, logger):
self.analysis = Analysis(
config.get_value("SHAREABLE", "trade_log_file_name"))
self.outfn = config.get_value("SHAREABLE", "stat_file_name")
self.logger = logger
def write(self):
report_text = ""
max_mo_length = len(str(self.analysis.max_markout_period()))
max_ticker_length = max([len(x)
for x in self.analysis.get_tickers()]) + 2
out_cols = [
"Markout Period", "Ticker", "Trade Count", "Return mu",
"Return tstat", "Return sharpe"
]
header = [
"{item:>{item_width}}".format(item=col, item_width=len(col))
for col in out_cols
]
report_text += "\t".join(header) + "\n"
for mo_period in self.analysis.get_mo_periods():
for ticker in self.analysis.get_tickers():
out = {
'Markout Period':
mo_period,
'Ticker':
ticker,
'Trade Count':
self.analysis.count(mo_period, ticker),
'Return mu':
"{:0.4f}".format(self.analysis.mean(mo_period, ticker)),
'Return tstat':
"{:0.2f}".format(self.analysis.tstat(mo_period, ticker)),
'Return sharpe':
"{:0.2f}".format(self.analysis.sharpe(mo_period, ticker))
}
output = []
for column in out_cols:
if column in out:
output.append("{item:>{item_width}}".format(
item=out[column], item_width=len(column)))
report_text += "\t".join(output) + "\n"
report_text += "\n"
with open(self.outfn, 'w') as f:
f.write(report_text)
self.logger.info("Report written to " + self.outfn)
def topics(request):
output = ''
if (request.GET.has_key('sort')):
freqTopics = ''
output = request.GET['sort']
time_diff = None #The time difference
if output == 'today':
time_diff = 1
elif output == 'lastweek':
time_diff = 7
else:
time_diff = 30
time2 = datetime.datetime.now()
time1 = time2 - timedelta(days=time_diff)
# freqTopics = Analysis.getFreqTopics(time1, time2, minFreq=10, k=params.maxFreqTopics)
freqTopics = Analysis.getFreqTopics(time1, time2, minFreq=0, k=params.maxFreqTopics)
time22 = time1
time11 = time22 - timedelta(days=time_diff) # to be used to find the old frequencies of current hot topics
print time1
print time2
print time11
print time22
freqTopics = Analysis.getTopicsFrequencyChanges(freqTopics, time11, time22) # find trendings for freq topic sets
return render_to_response('ui/topics.html', {'freqTopics' : freqTopics})
else:
mode = sys.argv[1]
datafiles = sys.argv[2:]
if 'full' in mode:
sys.setrecursionlimit(500) # watch out!
if 'single' in mode:
sys.setrecursionlimit(2000) # watch out!
try:
for datafile in datafiles:
instructions = Instructions()
parser = Parser(datafile, instructions)
analysis = Analysis(instructions)
if mode == 'debugnofilter':
logging.info("running debug mode with ipython on file %s..." % (datafile))
parser = Parser(datafile, instructions, [])
parser.run()
ipshell = IPShellEmbed()
ipshell()
if mode == 'debug':
logging.info("running debug mode with ipython on file %s..." % (datafile))
parser.run()
ipshell = IPShellEmbed()
def __init__(self, myParent):
self.analysis = Analysis()
if len(sys.argv)>1 and sys.argv[1] in ["CMD",'cmd','cl']:
self.Cmdline()
sys.exit()
self.myContainer = Frame(myParent,height=200,width=200)
self.myContainer.pack()
Label(self.myContainer, text="Choose an Element:").pack(anchor=W)
self.radio_element = StringVar()
element = [
(u"Ozone \u03bcg/m\xb3",'oz'),
(u"Particulate < 10 \u03bcg/m\xb3",'p10'),
(u"Particulate < 2.5 \u03bcg/m\xb3",'p2'),
]
for txt,val in element:
Radiobutton(self.myContainer,
text=txt,
indicatoron = 0,
width = 64,
padx = 20,
variable=self.radio_element,
command=self.showText,
value=val).pack(anchor=W)
self.checkbutton1 = StringVar()
self.C1 = Checkbutton(self.myContainer, text="Anand Vihar ",variable = self.checkbutton1,onvalue="AV",offvalue="")
self.checkbutton2 = StringVar()
self.C2 = Checkbutton(self.myContainer, text="Mandir Marg ",variable = self.checkbutton2,onvalue="MM",offvalue="")
self.checkbutton3 = StringVar()
self.C3 = Checkbutton(self.myContainer, text="Punjabi Bagh",variable = self.checkbutton3,onvalue="PB",offvalue="")
self.checkbutton4 = StringVar()
self.C4 = Checkbutton(self.myContainer, text="R.K Puram ",variable = self.checkbutton4,onvalue="RKP",offvalue="")
"""
#Button initials.
self.button1 = Button(self.myContainer,command=self.combine)
self.button1["text"]= "Execute"
"""
Label(self.myContainer, text="\nChoose Location(s):").pack(anchor = W)
self.C1.pack(anchor=W)
self.C2.pack(anchor=W)
self.C3.pack(anchor=W)
self.C4.pack(anchor=W)
"""Select yesterday, last 7 days, last Month , this month"""
self.time = None
self.time = StringVar()
date_options = [
(u"Yesterday",'y'),
(u"Last 7 Days",'l7'),
(u"This Month",'tm'),
(u"Last Month",'lm'),
(u"Since Beginning",'now')
]
for txt,val in date_options:
Radiobutton(self.myContainer,
text=txt,
indicatoron = 0,
width = 64,
padx = 20,
variable=self.time,
command=self.combine,
value=val).pack(anchor=W)
#Label(self.myContainer, text="\n").pack(anchor = W)
""""""
"""Enter date(optional)"""
#Not implemented
""""""
"""Execute Button"""
#self.button1.pack()
self.text = None
self.ozone = None
self.particulate = None
with open('ozone.txt','r') as f:
self.ozone = f.read()
with open('particulate.txt','r') as f:
self.particulate = f.read()
s = urllib.urlopen(SITE_JR + fileName)
if s.getcode() != 200:
return
lines = s.readlines()
print u"%s => %s" % (fileName,catName)
ret = "{|class='wikitable sortable'\n"
ret += "!Titre!!Nb links!!Nb pages traduites!!tpl !! en page !! de page !! es page !! models !! admisssible \n"
for l in lines:
ret += "|-\n"
ret += l
ret +="|-\n"
ret+= "|}"
p = Page(site, page + "/analysis")
p.edit(text = ret, summary=str(len(lines)) + " articles à adopter",bot=True)
a = Analysis(site)
a.run()
YEAR = [2013, 2014, 2015, 2016, 2017]
MONTH = [u"janvier", u"février", u"mars", u"avril", u"mai", u"juin", u"juillet", u"août", u"septembre", u"octobre", u"novembre", u"décembre"]
# orphelins
for y in YEAR:
m =1
for mon in MONTH:
catName = u"Catégorie:Article orphelin depuis %s %d" %(mon, y)
fileName= u"orph_%d-%02d.arch" % (y, m)
try:
printPageFromFile(catName, fileName)
except:
print "problem with %s" % catName
class Gui(object):
def __init__(self, myParent):
self.analysis = Analysis()
if len(sys.argv)>1 and sys.argv[1] in ["CMD",'cmd','cl']:
self.Cmdline()
sys.exit()
self.myContainer = Frame(myParent,height=200,width=200)
self.myContainer.pack()
Label(self.myContainer, text="Choose an Element:").pack(anchor=W)
self.radio_element = StringVar()
element = [
(u"Ozone \u03bcg/m\xb3",'oz'),
(u"Particulate < 10 \u03bcg/m\xb3",'p10'),
(u"Particulate < 2.5 \u03bcg/m\xb3",'p2'),
]
for txt,val in element:
Radiobutton(self.myContainer,
text=txt,
indicatoron = 0,
width = 64,
padx = 20,
variable=self.radio_element,
command=self.showText,
value=val).pack(anchor=W)
self.checkbutton1 = StringVar()
self.C1 = Checkbutton(self.myContainer, text="Anand Vihar ",variable = self.checkbutton1,onvalue="AV",offvalue="")
self.checkbutton2 = StringVar()
self.C2 = Checkbutton(self.myContainer, text="Mandir Marg ",variable = self.checkbutton2,onvalue="MM",offvalue="")
self.checkbutton3 = StringVar()
self.C3 = Checkbutton(self.myContainer, text="Punjabi Bagh",variable = self.checkbutton3,onvalue="PB",offvalue="")
self.checkbutton4 = StringVar()
self.C4 = Checkbutton(self.myContainer, text="R.K Puram ",variable = self.checkbutton4,onvalue="RKP",offvalue="")
"""
#Button initials.
self.button1 = Button(self.myContainer,command=self.combine)
self.button1["text"]= "Execute"
"""
Label(self.myContainer, text="\nChoose Location(s):").pack(anchor = W)
self.C1.pack(anchor=W)
self.C2.pack(anchor=W)
self.C3.pack(anchor=W)
self.C4.pack(anchor=W)
"""Select yesterday, last 7 days, last Month , this month"""
self.time = None
self.time = StringVar()
date_options = [
(u"Yesterday",'y'),
(u"Last 7 Days",'l7'),
(u"This Month",'tm'),
(u"Last Month",'lm'),
(u"Since Beginning",'now')
]
for txt,val in date_options:
Radiobutton(self.myContainer,
text=txt,
indicatoron = 0,
width = 64,
padx = 20,
variable=self.time,
command=self.combine,
value=val).pack(anchor=W)
#Label(self.myContainer, text="\n").pack(anchor = W)
""""""
"""Enter date(optional)"""
#Not implemented
""""""
"""Execute Button"""
#self.button1.pack()
self.text = None
self.ozone = None
self.particulate = None
with open('ozone.txt','r') as f:
self.ozone = f.read()
with open('particulate.txt','r') as f:
self.particulate = f.read()
def combine(self):
self.buildCmd()
try:
self.forget()
except:
return
def forget(self):
self.text.destroy()
self.text = None
def showText(self):
"""After selecting an element have to select location(s)"""
if self.v.get()=="oz" and self.text==None:
self.text = Text(self.myContainer,width=67, height=15)
self.text.configure(state='normal')
self.text.insert('1.0', self.ozone)
self.text.configure(state='disabled')
self.text.pack()
elif (self.v.get()=="p2" or self.v.get()=="p10") and self.text==None:
self.text = Text(self.myContainer,width=67, height=15)
self.text.configure(state='normal')
self.text.insert('1.0', self.particulate)
self.text.configure(state='disabled')
self.text.pack()
def buildCmd(self):
interval = self.time.get()
if interval=='':
interval = 'now'
s = "get %s in %s %s %s %s on %s" % (self.radio_element.get(),self.checkbutton1.get(), self.checkbutton2.get(),self.checkbutton3.get(),self.checkbutton4.get(),interval)
query = s.split()
#print query
if len(query)<6 or query[1]=='in': #Need to check for locations missing
return
if query[0]=='get':
self.analysis.get_data(query[1:])
def Cmdline(self):
#q = {"date":datetime(2015,04,23,15,10)}
#find_data(q)[0]
while True:
print ""
#print "Elements = oz: ozone , p10: particulate < 10 , p2: particulate < 2.5"
#print "Location = RK: RK Puram , MM: Mandir Marg , AV: Anand Vihar, PB: Punjabi Bagh"
#print "Time = now: Since Beginning , y: Yesterday, l7: Last 7 Days, tm: This Month, lm: Last Month"
#print "Quit = q"
print "Enter Query-->",
try:
s = raw_input()
if s in ["quit","q",'exit']:
break
query = s.split()
if query[0]=='get':
check = self.analysis.get_data(query[1:])
if check==0:
print "Error in Query"
continue
except:
continue
print "Bye"
pass #print report query = 'Reuters' #time1 = '2012-04-20 00:00:00' #time2 = '2013-04-25 08:00:00' #time11 = '2012-04-25 08:00:00' #time22 = '2013-04-30 00:00:00' time2 = datetime.datetime.now() time1 = time2 - timedelta(days=20) time22 = time1 time11 = time22 - timedelta(days=20) print 'getFreqTopics:' print Analysis.getFreqTopics(time1, time2, minFreq=10, k=0) print '\ngetFreqTopicSets:' print Analysis.getFreqTopicSets(query, time1, time2) print '\ngetTopicFrequency:' print Analysis.getTopicFrequency(query, time1, time2) print '\ngetHotTopics:' topics = Analysis.getHotTopics(10, time1, time2) print topics print '\ngetTopicsFrequencyChanges:' print Analysis.getTopicsFrequencyChanges(topics, time11, time22)
def __init__(self, config, logger):
self.analysis = Analysis(config.get_value("SHAREABLE", "trade_log_file_name"))
self.outfn = config.get_value("SHAREABLE", "stat_file_name")
self.logger = logger
def index(request):
freqTopicSets = ''
query = ''
freqTopics = ''
time2 = datetime.datetime.now()
time1 = time2 - timedelta(days=1)
print('timing: before freqTopics')
print(datetime.datetime.now())
freqTopics = Analysis.getFreqTopics(time1, time2, minFreq=0, k=params.maxFreqTopics)
time22 = time1
time11 = time22 - timedelta(days=2*params.days) # to be used to find the old frequencies of current hot topics
freqTopics = Analysis.getTopicsFrequencyChanges(freqTopics, time11, time22) # find trendings for freq topic sets
#pdb.set_trace() how to do debugging
#print('timing: after freqTopics')
#print(datetime.datetime.now())
#if request.method == 'GET' : # If form is submitted
form = searchForm(request.GET)
tag = ''
linkSets = []
topicSet_linkSets = []
if form.is_valid():
if(request.GET.has_key('input')):
print('timing: before getFreqTopicSets')
print(datetime.datetime.now())
freqTopicSets = Analysis.getFreqTopicSets(request.GET['input'], time1, time2)
print('timing: after getFreqTopicSets')
print(datetime.datetime.now())
for topicSet in freqTopicSets:
tag_ids = []
links = []
#cuz example topicSet for a query like Microsoft = (['Blog', 'Android'], 4) so make into ['Blog', 'Android', 'Microsoft']
topics = topicSet[0]
topics.append(str(request.GET['input']))
for topic in topics:
tag = Tag.findByTitle(topic) #findByTitle returns multiple rows representing multiple tags, so the last [0][0] means get the first attribute (ie id) or the first tag in rows
#pdb.set_trace()
tag_id = tag[0][0]
tag_id = str(tag_id).strip('L') #seems like we get id's like '3167L' strip the L
tag_ids.append(tag_id)
#print("in views.py:")
print(topicSet)
#pdb.set_trace()
#print('====')
#print(datetime.datetime.now())
feeditems = feeditem.Feeditem.findByTags(tag_ids, time1, time2)
#print(datetime.datetime.now())
for item in feeditems:
print item[1]
t1 = unicode(unicode(item[1],'utf-8', errors='ignore')) # to ignore non utf-8 chars
t2 = unicode(item[3])
links.append((t1,t2))
linkSets.append(copy.copy(links))
topicSet_linkSets = zip(freqTopicSets, linkSets)
else:
form = searchForm()
if(request.GET.has_key('input')):
query = request.GET['input']
else:
qeury = None
return render_to_response('ui/index.html', {'form' : form, 'freqTopicSets':freqTopicSets, 'query':
query, 'freqTopics':freqTopics, 'linkSets':linkSets, 'topicSet_linkSets':topicSet_linkSets[::-1] })
def __init__(self):
Analysis.__init__(self)
self.packetList = []
