def test_decrypt_files(self):
sleep(1)
P = PyHasher(PATH=getcwd(), pattern="**/TU/_CRYPT_*.txt", passw=_TEST_PASS, salt=_TEST_SALT, mode="decrypt")
P.routine()
B = Aggregator(pattern="**/TU/*DECRYPTED_1__CRYPT_1*.txt", path_to_files=getcwd())
C = Aggregator(pattern="**/TU/test_file.txt", path_to_files=getcwd())
self.assertEqual("_DECRYPTED_1__CRYPT_1_test_file.txt" in B.matching_list[0], second=True)
with open(B.matching_list[0], "r") as b:
with open(C.matching_list[0], 'r') as c:
self.assertEqual(b.read(), c.read())
c.close()
b.close()
def test_get_users(self):
gm_client = gearman.GearmanClient(['localhost:4730'])
agg = Aggregator(gm_client, None)
user_data = agg.get_users()
for i in range(len(user_data)):
del user_data[i]['_id']
if 'words' in user_data[i]:
del user_data[i]['words']
expected_response = [
{
u'username': u'iandioch',
u'subscribed_feeds': [
u'https://news.ycombinator.com/rss'
],
},
{
u'username': u'sully',
u'subscribed_feeds': [
u'https://news.ycombinator.com/rss',
u'http://spritesmods.com/rss.php',
u'http://dave.cheney.net/feed'
],
},
{
u'username': u'theotherguys',
u'subscribed_feeds': [
u'https://news.ycombinator.com/rss',
u'http://spritesmods.com/rss.php'
],
}]
self.assertEqual(user_data, expected_response)
def test_find_near():
db = DB()
aggregator = Aggregator(db)
car_id_1 = aggregator.login_car()
car_id_2 = aggregator.login_car()
car_id_3 = aggregator.login_car()
car_1 = db.get_car(car_id_1)
car_2 = db.get_car(car_id_2)
car_3 = db.get_car(car_id_3)
aggregator.new_position_car(car_id_1, 10, 10)
aggregator.new_position_car(car_id_2, 20, 20)
aggregator.new_position_car(car_id_3, 30, 30)
assert set(db.find_near(Point(5, 5), 2)) == set([car_1, car_2])
aggregator.logout_car(car_id_1, "Test reason")
aggregator.logout_car(car_id_2, "Test reason")
aggregator.logout_car(car_id_3, "Test reason")
with pytest.raises(CarNotFound):
db.get_car(car_id_1)
with pytest.raises(CarNotFound):
db.get_car(car_id_2)
with pytest.raises(CarNotFound):
db.get_car(car_id_3)
def test_put_g2g(self):
gm_client = gearman.GearmanClient(['localhost:4730'])
agg = Aggregator(gm_client, None)
# add_request = bson.BSON.encode({
# 'database':'feedlark',
# 'collecion':'g2g',
# 'data':{
# 'username':'******',
# 'test_parameter':'NOLO'
# }
# })
# gm_client.submit_job('db-add',str(add_request))
test_document = {
'username': '******',
'test_parameter': 'YOLO',
}
agg.put_g2g('iandioch', test_document)
get_request = bson.BSON.encode({
'database': 'feedlark',
'collection': 'g2g',
'query': {
'username': '******',
},
'projection': {
'test_parameter': 1,
},
})
g2g_data = gm_client.submit_job('db-get', str(get_request)).result
self.assertEqual(
bson.BSON(g2g_data).decode()['docs'][0]['test_parameter'], 'YOLO')
def group_by(self, time_unit):
"""
Get results by time slice ('days', 'hours', 'minutes', 'seconds').
Returns a generator of times and tweet-generators, like this:
Example:
########
for time, tweets in collection.group_by('minutes'):
for tweet in tweets:
# do something
---------------------------------------------------
Also supports top_x() methods like this:
Example:
########
collection.since(datetime(2015,6,1)).group_by('days').top_user_locations(n=5)
>
# London London, UK Manchester Scotland UK
# 2015-06-1 4 2 1 1 2
# 2015-06-2 11 4 9 3 3
# 2015-06-3 14 1 5 NaN 4
# 2015-06-4 17 1 5 1 6
# 2015-06-5 10 3 3 3 3
"""
return Aggregator(self, time_unit=time_unit)
def __init__(self):
self.aggregator = Aggregator() # to retrive server logs
self.server_set = set(util.retrieve_server_list()) # [server_ip, ]
self.content_set = set()
self.replica_map = {} # { file_uuid: servers that store the file }
self.replication_task = []
self.last_timestamp = 0 # the time stamp of last update
def test_car_login_logout():
db = DB()
aggregator = Aggregator(db)
car_id = aggregator.login_car()
car = db.get_car(car_id)
aggregator.logout_car(car_id, "Test reason")
with pytest.raises(CarNotFound):
db.get_car(car_id)
def __init__(self, dataset):
self.aggregator = Aggregator()
self.dataset = dataset
self.target_index, target = self.__choose_random_sample(dataset)
self.target = copy.deepcopy(target)
#self.target.set_mesh_and_parts_colour([1, 0, 0])
self.set_colour_coded_sources(
dataset) ############# ADD TO RESET TARGET?? ##############
def main():
initialize_log()
logging.info("Starting aggregator")
config_params = parse_config_params()
aggregator = Aggregator(config_params['source_queue'],
config_params['reducer_queue'],
config_params['key'])
aggregator.start()
def run_experiment(self):
with open('{}/anomaly.json'.format(DATA_FOLDER), 'r') as f:
an_data = json.load(f)
agg = Aggregator(an_data)
incidents, relevance = agg.build_incidents_report()
metrics_df = pd.read_csv(
'{}/metrics_0_filter.csv'.format(DATA_FOLDER))
for key, item in incidents.items():
self.__do_report(metrics_df, an_data, key, item)
def __init__(self, **kwargs):
self.reportData = None
#insert your model class names here. Replace 'DummyModel' with your ai model class name
self.models_aggregator = Aggregator(
[nnModel.NNModel, foodRandomForest.FoodRandomForest],
[[
'nnmodels/nnmodel0.pkl', 'nnmodels/nnmodel1.pkl',
'nnmodels/nnmodel2.pkl', 'nnmodels/nnmodel3.pkl',
'nnmodels/nnmodel4.pkl', 'nnmodels/nnmodel5.pkl',
'nnmodels/nnmodel6.pkl', 'nnmodels/nnmodel7.pkl',
'nnmodels/nnmodel8.pkl', 'nnmodels/nnmodel9.pkl',
'nnmodels/nnmodel10.pkl', 'nnmodels/nnmodel11.pkl',
'nnmodels/nnmodel12.pkl', 'nnmodels/nnmodel13.pkl',
'nnmodels/nnmodel14.pkl', 'nnmodels/nnmodel15.pkl',
'nnmodels/nnmodel16.pkl', 'nnmodels/nnmodel17.pkl',
'nnmodels/nnmodel18.pkl', 'nnmodels/nnmodel19.pkl',
'nnmodels/nnmodel20.pkl', 'nnmodels/nnmodel21.pkl',
'nnmodels/nnmodel22.pkl', 'nnmodels/nnmodel23.pkl',
'nnmodels/nnmodel24.pkl', 'nnmodels/nnmodel25.pkl',
'nnmodels/nnmodel26.pkl', 'nnmodels/nnmodel27.pkl',
'nnmodels/nnmodel28.pkl', 'nnmodels/nnmodel29.pkl',
'nnmodels/nnmodel30.pkl'
],
[
'rf-models/randomforest0.joblib',
'rf-models/randomforest1.joblib',
'rf-models/randomforest2.joblib',
'rf-models/randomforest3.joblib',
'rf-models/randomforest4.joblib',
'rf-models/randomforest5.joblib',
'rf-models/randomforest6.joblib',
'rf-models/randomforest7.joblib',
'rf-models/randomforest8.joblib',
'rf-models/randomforest9.joblib',
'rf-models/randomforest10.joblib',
'rf-models/randomforest11.joblib',
'rf-models/randomforest12.joblib',
'rf-models/randomforest13.joblib',
'rf-models/randomforest14.joblib',
'rf-models/randomforest15.joblib',
'rf-models/randomforest16.joblib',
'rf-models/randomforest17.joblib',
'rf-models/randomforest18.joblib',
'rf-models/randomforest19.joblib',
'rf-models/randomforest20.joblib',
'rf-models/randomforest21.joblib',
'rf-models/randomforest22.joblib',
'rf-models/randomforest23.joblib',
'rf-models/randomforest24.joblib',
'rf-models/randomforest25.joblib',
'rf-models/randomforest26.joblib',
'rf-models/randomforest27.joblib',
'rf-models/randomforest28.joblib',
'rf-models/randomforest29.joblib'
]])
super(UltraProcessedFoodApp, self).__init__(**kwargs)
async def update_mocks():
directory = Path(__file__).parent
aggr = Aggregator(rapidapi_key=RAPIDAPI_KEY)
for source in aggr._sources:
source_name = type(source).__name__
file_path = directory / f'mock_response_{source_name}.txt'
response = await source.load_data()
with file_path.open('w') as file:
file.write(response)
def __init__(self):
self.aggregator = Aggregator() # to retrive server logs
self.client_set = set([]) # [client_ip, ]
self.server_set = set(util.retrieve_server_list()) # [server_ip, ]
self.content_set = set([]) # [uuid, ]
self.access_map = {} # {uuid: {client_ip: num_request}}
self.replica_map = {} # {uuid: {server_ip: num_replica}}
self.last_timestamp = 0 # the timestamp of last update
self.requests_per_replica = 3
self.uuid_to_server = None
async def test_update():
aggr = Aggregator()
await aggr.update()
aggr._rapidapi.update.assert_called_once()
aggr._github.update.assert_called_once()
aggr._stopcorona.update.assert_called_once()
with patch.object(RapidapiSource, 'update', _async_raise(ReadTimeout)):
aggr = Aggregator()
await aggr.update()
logging.error.assert_called_with('ReadTimeout in source update')
with patch.object(GithubSource, 'update', _async_raise(ConnectTimeout)):
aggr = Aggregator()
await aggr.update()
logging.error.assert_called_with('ConnectTimeout in source update')
with patch.object(GithubSource, 'update', _async_raise(ZeroDivisionError)):
aggr = Aggregator()
await aggr.update()
logging.exception.assert_called_once()
def __init__(self, line, debug):
reader = EffectReader(line)
# reader = TraceReader(line)
reader.parse_trace()
self.code_size = len(reader.code)
signal.signal(signal.SIGALRM, handler)
signal.alarm(15)
# print(reader.signature)
optimizer = Aggregator(reader.code)
ExpressionExecutor(reader, optimizer, debug)
signal.alarm(0)
def main():
set_logging()
bot = aiogram.Bot(BOT_TOKEN)
dispatcher = aiogram.Dispatcher(bot)
aggregator = Aggregator(rapidapi_key=RAPIDAPI_KEY)
asyncio.run(aggregator.load_sources())
dispatcher.loop.create_task(aggregator.update_periodically())
communicator = Communicator(aggregator)
register_handlers(communicator, dispatcher)
run(dispatcher)
def main() -> None:
args_parser = argparse.ArgumentParser()
args_parser.add_argument("--cache-dir",
dest="cache_dir",
metavar="DIR",
type=str,
default=".cache")
args_parser.add_argument("--export-dir",
dest="export_dir",
metavar="DIR",
type=str,
default=".export")
args_parser.add_argument(
"--templates-dir",
dest="templates_dir",
metavar="DIR",
type=str,
default="templates",
)
args_parser.add_argument("--base-url",
dest="base_url",
metavar="URL",
type=str)
args_parser.add_argument(
"--podcasts-json",
dest="podcasts_json",
metavar="FILE",
type=str,
default="podcasts.json",
)
args_parser.add_argument("--clear-cache",
dest="clear_cache",
action="store_true")
args_parser.add_argument("--keep-feeds",
dest="keep_feeds",
action="store_true")
args = args_parser.parse_args()
a = Aggregator(
podcasts_json=args.podcasts_json,
cache_dir=args.cache_dir,
export_dir=args.export_dir,
templates_dir=args.templates_dir,
base_url=args.base_url if args.base_url else "file://{}".format(
os.path.abspath(args.export_dir)),
)
if args.clear_cache:
a.clear_cache()
a.sync(keep_feeds=args.keep_feeds)
a.export()
def main():
nasdaq = StockMarket("NASDAQ", ["AAPL", "CSCO", "MSFT", "GOOG"])
newyork = StockMarket("NYSE", ["IBM", "HPQ", "BP"])
agg = Aggregator()
agg.add_market(nasdaq)
agg.add_symbols(nasdaq.symbols)
agg.add_market(newyork)
agg.add_symbols(newyork.symbols)
print("aggregated symbols:", agg.symbols)
view = Viewer()
view.aggregator(agg, ["IBM", "AAPL", "MSFT"])
view.print_quotes()
def run_experiment(self):
agg = Aggregator(255, 10)
with open('{}/anomaly.json'.format(DATA_FOLDER), 'r') as f:
an_data = json.load(f)
incidents, relevance = agg.build_incidents_report(an_data)
metrics_df = pd.read_csv('{}/metrics_0_filter.csv'.format(DATA_FOLDER))
for key, item in incidents.items():
image_file = '{}_viz.png'.format(key)
visualisation = VisualizeReports(metrics_df, an_data, item)
visualisation.visualize_with_siblings('{}/{}'.format(SAMPLES_FOLDER, image_file))
self.__upload_file('{}/{}'.format(SAMPLES_FOLDER, image_file), image_file)
self.__run_incident_report_buttons(key, image_file)
def go(b1, b2, b3, p, output):
"""
Launches the triangulation for the given input file and writes result in new CSV
:return:
"""
tab = Aggregator().aggregate(b1, b2, b3, p)
with open(output, 'w') as csvfile:
csv_writer = csv.writer(csvfile,
delimiter=',',
quotechar='|',
quoting=csv.QUOTE_MINIMAL)
for row in tab:
csv_writer.writerow(row)
return 0
def __init__(self, args, n_entity, n_relation,adj_entity,adj_relation): # 初始化使用 args n_entity n_relation
super(RippleNet, self).__init__()
self._parse_args(args, n_entity, n_relation)
self.entity_emb = nn.Embedding(self.n_entity, self.dim)
self.relation_emb = nn.Embedding(self.n_relation, self.dim * self.dim)
self.relation_emb_GCN = nn.Embedding(self.n_relation,self.dim)
self.transform_matrix = nn.Linear(self.dim, self.dim, bias=False)
self.criterion = nn.BCELoss()
self.transformerEncoderLayer = nn.TransformerEncoderLayer(d_model=self.dim, nhead=self.n_head, dim_feedforward=self.feed_f_dim)
self.transformerEncoder = nn.TransformerEncoder(num_layers=1, encoder_layer=self.transformerEncoderLayer)
self.pooling = nn.AvgPool2d
self.linear = nn.Linear(in_features=2 * self.dim, out_features=self.dim)
self.adj_entity = adj_entity
self.adj_relation = adj_relation
self.aggregator = Aggregator(self.batch_size, self.dim, args.aggregator)
def main():
nasdaq=StockMarket("NASDAQ", ["AAPL", "CSCO", "MSFT", "GOOG"])
newyork=StockMarket("NYSE", ["IBM", "HPQ", "BP"])
agg=Aggregator()
agg.add_symbols(nasdaq.symbols())
agg.add_symbols(newyork.symbols())
print("aggregated symbols:", agg.available_symbols())
nasdaq.listener(agg)
newyork.listener(agg)
view=Viewer()
agg.view(view, ["IBM", "AAPL", "MSFT"])
print("")
while True:
nasdaq.generate()
newyork.generate()
time.sleep(0.5)
def __init__(self, num_user, num_ent, num_rel, kg, args, device):
super(KGCN, self).__init__()
self.num_user = num_user
self.num_ent = num_ent
self.num_rel = num_rel
self.n_iter = args.n_iter
self.batch_size = args.batch_size
self.dim = args.dim
self.n_neighbor = args.neighbor_sample_size
self.kg = kg
self.device = device
self.aggregator = Aggregator(self.batch_size, self.dim,
args.aggregator)
self._gen_adj()
self.usr = torch.nn.Embedding(num_user, args.dim)
self.ent = torch.nn.Embedding(num_ent, args.dim)
self.rel = torch.nn.Embedding(num_rel, args.dim)
def main():
# capture the config path from the run arguments
# then process the json configuration file
try:
args = get_args()
config = process_config(args.config)
except:
print("missing or invalid arguments")
exit(0)
# create the experiments dirs
create_dirs([config['result_dir'], config['checkpoint_dir'], config['checkpoint_dir_lstm']])
# save the config in a txt file
save_config(config)
# create tensorflow session
sessions = []
data = []
model_vaes = []
vae_trainers = []
lstm_models = []
model_vae_global = VAEmodel(config, "Global")
sess_global = tf.Session(config=tf.ConfigProto())
for i in range(1, 10):
sess = tf.Session(config=tf.ConfigProto())
sessions.append(sess)
data.append(generator_fl(config, i))
model_vaes.append(VAEmodel(config, "Client{}".format(i)))
model_vaes[-1].load(sessions[-1])
vae_trainers.append(vaeTrainer(sessions[-1], model_vaes[-1], data[-1], config))
lstm_models.append(lstmKerasModel("Client{}".format(i), config))
model_vae_global.load(sess_global)
trainer_vae_global = vaeTrainer(sess_global, model_vae_global, data[0], config)
lstm_model_global = lstmKerasModel("Global", config)
client_weights = [0.1] * 8
client_weights.append(0.2)
aggregator = Aggregator(vae_trainers, trainer_vae_global, lstm_models, lstm_model_global, config, client_weights)
aggregator.aggregate_vae()
aggregator.aggregate_lstm()
def test_check_operator(self):
"""
This test is not working because the Exception is not caught by the context : TO BE CORRECTED
:return: None
"""
with self.assertRaises(ValueError): Aggregator(self.comp_parent, [self.comp_1, self.comp_3], 'Invalid_Code')
def test_div(self):
agg = Aggregator(self.comp_parent, [self.comp_1, self.comp_3], Operator.DIV)
agg.aggregate()
self.assertEqual(self.comp_parent.data.equals(pd.Series([0.5, 1, 1.5, 2, 2.5], index=self.idx_1)), True)
def test_mul(self):
agg = Aggregator(self.comp_parent, [self.comp_1, self.comp_2, self.comp_3], Operator.MULT)
agg.aggregate()
self.assertEqual(self.comp_parent.data.equals(pd.Series([10, 16, 18, 16, 10], index=self.idx_1)), True)
def test_sub(self):
agg = Aggregator(self.comp_parent, [self.comp_2, self.comp_3], Operator.SUB)
agg.aggregate()
self.assertEqual(self.comp_parent.data.equals(pd.Series([3, 2, 1, 0, -1], index=self.idx_1)), True)
def test_add(self):
print("Operator est de type: ", type(Operator))
agg = Aggregator(self.comp_parent, [self.comp_1, self.comp_2, self.comp_3], Operator.ADD)
agg.aggregate()
self.assertEqual(self.comp_parent.data.equals(pd.Series([8, 8, 8, 8, 8], index=self.idx_1)), True)
from flask import Flask
from aggregator import Aggregator
import asyncio
import websockets
from flask_cors import CORS
from flask_socketio import SocketIO, send, emit
from reddit_client import RedditClient
PERIOD = 5
TOP = 50
app = Flask(__name__)
socketio = SocketIO(app, cors_allowed_origins="*")
CORS(app)
aggregator = Aggregator(PERIOD, TOP)
client = RedditClient(aggregator, socketio)
client.start_streaming()
@socketio.on('test')
def handle_message(data):
emit("res", "from the server")
if __name__ == '__main__':
socketio.run(app)
