def test_get_users(self):
gm_client = gearman.GearmanClient(['localhost:4730'])
agg = Aggregator(gm_client, None)
user_data = agg.get_users()
for i in range(len(user_data)):
del user_data[i]['_id']
if 'words' in user_data[i]:
del user_data[i]['words']
expected_response = [
{
u'username': u'iandioch',
u'subscribed_feeds': [
u'https://news.ycombinator.com/rss'
],
},
{
u'username': u'sully',
u'subscribed_feeds': [
u'https://news.ycombinator.com/rss',
u'http://spritesmods.com/rss.php',
u'http://dave.cheney.net/feed'
],
},
{
u'username': u'theotherguys',
u'subscribed_feeds': [
u'https://news.ycombinator.com/rss',
u'http://spritesmods.com/rss.php'
],
}]
self.assertEqual(user_data, expected_response)
def __init__(self):
self.aggregator = Aggregator() # to retrive server logs
self.server_set = set(util.retrieve_server_list()) # [server_ip, ]
self.content_set = set()
self.replica_map = {} # { file_uuid: servers that store the file }
self.replication_task = []
self.last_timestamp = 0 # the time stamp of last update
def test_put_g2g(self):
gm_client = gearman.GearmanClient(['localhost:4730'])
agg = Aggregator(gm_client, None)
# add_request = bson.BSON.encode({
# 'database':'feedlark',
# 'collecion':'g2g',
# 'data':{
# 'username':'******',
# 'test_parameter':'NOLO'
# }
# })
# gm_client.submit_job('db-add',str(add_request))
test_document = {
'username': '******',
'test_parameter': 'YOLO',
}
agg.put_g2g('iandioch', test_document)
get_request = bson.BSON.encode({
'database': 'feedlark',
'collection': 'g2g',
'query': {
'username': '******',
},
'projection': {
'test_parameter': 1,
},
})
g2g_data = gm_client.submit_job('db-get', str(get_request)).result
self.assertEqual(
bson.BSON(g2g_data).decode()['docs'][0]['test_parameter'], 'YOLO')
def main():
initialize_log()
logging.info("Starting aggregator")
config_params = parse_config_params()
aggregator = Aggregator(config_params['source_queue'],
config_params['reducer_queue'],
config_params['key'])
aggregator.start()
def __init__(self, dataset):
self.aggregator = Aggregator()
self.dataset = dataset
self.target_index, target = self.__choose_random_sample(dataset)
self.target = copy.deepcopy(target)
#self.target.set_mesh_and_parts_colour([1, 0, 0])
self.set_colour_coded_sources(
dataset) ############# ADD TO RESET TARGET?? ##############
def test_car_login_logout():
db = DB()
aggregator = Aggregator(db)
car_id = aggregator.login_car()
car = db.get_car(car_id)
aggregator.logout_car(car_id, "Test reason")
with pytest.raises(CarNotFound):
db.get_car(car_id)
def test_car_login_logout():
db = DB()
aggregator = Aggregator(db)
car_id = aggregator.login_car()
car = db.get_car(car_id)
aggregator.logout_car(car_id, "Test reason")
with pytest.raises(CarNotFound):
db.get_car(car_id)
def __init__(self, binary):
"""
构建器初始化
:param binary:字节码
"""
logging.info("构建器初始化")
Aggregator.__init__(self, binary) # 聚合初始化
for func in self.get_all_functions(): # 遍历函数
self.__analyze_function(func)
def run_experiment(self):
with open('{}/anomaly.json'.format(DATA_FOLDER), 'r') as f:
an_data = json.load(f)
agg = Aggregator(an_data)
incidents, relevance = agg.build_incidents_report()
metrics_df = pd.read_csv(
'{}/metrics_0_filter.csv'.format(DATA_FOLDER))
for key, item in incidents.items():
self.__do_report(metrics_df, an_data, key, item)
def run_aggregator(self, items_queue, sender_queue):
aggregator_config = self.config.get("aggregator")
aggregator_config["log_format"] = self.log_format
aggregator = Aggregator(
items_queue=items_queue, sender_queue=sender_queue, shutdown=self.shutdown_event, config=aggregator_config
)
self.processes.append(aggregator)
aggregator.start()
def main(): formatter = Tap(Stdout()) global_env = init(formatter) agg = Aggregator() agg.add_files(option.targets) runner = ScenarioRunner(agg, formatter) runner.run(global_env)
def __init__(self):
self.aggregator = Aggregator() # to retrive server logs
self.client_set = set([]) # [client_ip, ]
self.server_set = set(util.retrieve_server_list()) # [server_ip, ]
self.content_set = set([]) # [uuid, ]
self.access_map = {} # {uuid: {client_ip: num_request}}
self.replica_map = {} # {uuid: {server_ip: num_replica}}
self.last_timestamp = 0 # the timestamp of last update
self.requests_per_replica = 3
self.uuid_to_server = None
def test_decrypt_files(self):
sleep(1)
P = PyHasher(PATH=getcwd(), pattern="**/TU/_CRYPT_*.txt", passw=_TEST_PASS, salt=_TEST_SALT, mode="decrypt")
P.routine()
B = Aggregator(pattern="**/TU/*DECRYPTED_1__CRYPT_1*.txt", path_to_files=getcwd())
C = Aggregator(pattern="**/TU/test_file.txt", path_to_files=getcwd())
self.assertEqual("_DECRYPTED_1__CRYPT_1_test_file.txt" in B.matching_list[0], second=True)
with open(B.matching_list[0], "r") as b:
with open(C.matching_list[0], 'r') as c:
self.assertEqual(b.read(), c.read())
c.close()
b.close()
def main():
nasdaq = StockMarket("NASDAQ", ["AAPL", "CSCO", "MSFT", "GOOG"])
newyork = StockMarket("NYSE", ["IBM", "HPQ", "BP"])
agg = Aggregator()
agg.add_market(nasdaq)
agg.add_symbols(nasdaq.symbols)
agg.add_market(newyork)
agg.add_symbols(newyork.symbols)
print("aggregated symbols:", agg.symbols)
view = Viewer()
view.aggregator(agg, ["IBM", "AAPL", "MSFT"])
view.print_quotes()
def main():
formatter = Tap(Stdout())
global_env = init(formatter)
if option.targets:
agg = Aggregator()
agg.add_files(option.targets)
runner = ScenarioRunner(agg, formatter)
failed = runner.run(global_env)
else:
import console
console.run(global_env)
teardown()
if failed:
sys.exit(1)
def main():
set_logging()
bot = aiogram.Bot(BOT_TOKEN)
dispatcher = aiogram.Dispatcher(bot)
aggregator = Aggregator(rapidapi_key=RAPIDAPI_KEY)
asyncio.run(aggregator.load_sources())
dispatcher.loop.create_task(aggregator.update_periodically())
communicator = Communicator(aggregator)
register_handlers(communicator, dispatcher)
run(dispatcher)
def run_experiment(self):
agg = Aggregator(255, 10)
with open('{}/anomaly.json'.format(DATA_FOLDER), 'r') as f:
an_data = json.load(f)
incidents, relevance = agg.build_incidents_report(an_data)
metrics_df = pd.read_csv('{}/metrics_0_filter.csv'.format(DATA_FOLDER))
for key, item in incidents.items():
image_file = '{}_viz.png'.format(key)
visualisation = VisualizeReports(metrics_df, an_data, item)
visualisation.visualize_with_siblings('{}/{}'.format(SAMPLES_FOLDER, image_file))
self.__upload_file('{}/{}'.format(SAMPLES_FOLDER, image_file), image_file)
self.__run_incident_report_buttons(key, image_file)
def run_scenario(global_env):
formatter = global_env['formatter']
failed = False
try:
agg = Aggregator()
agg.add_filter(TagFilter(option.tags))
agg.add_files(option.targets)
except ScenarioLoadError as e:
formatter.writer.write("{0}({1}):\n {2}\n\n".format(e.__class__.__name__, e.error, e.filename))
formatter.writer.write(util.unicode(e) + "\n")
failed = True
runner = ScenarioRunner(agg, formatter)
failed = failed or runner.run(global_env)
return int(failed)
def group_by(self, time_unit):
"""
Get results by time slice ('days', 'hours', 'minutes', 'seconds').
Returns a generator of times and tweet-generators, like this:
Example:
########
for time, tweets in collection.group_by('minutes'):
for tweet in tweets:
# do something
---------------------------------------------------
Also supports top_x() methods like this:
Example:
########
collection.since(datetime(2015,6,1)).group_by('days').top_user_locations(n=5)
>
# London London, UK Manchester Scotland UK
# 2015-06-1 4 2 1 1 2
# 2015-06-2 11 4 9 3 3
# 2015-06-3 14 1 5 NaN 4
# 2015-06-4 17 1 5 1 6
# 2015-06-5 10 3 3 3 3
"""
return Aggregator(self, time_unit=time_unit)
def __init__(self, **kwargs):
self.reportData = None
#insert your model class names here. Replace 'DummyModel' with your ai model class name
self.models_aggregator = Aggregator(
[nnModel.NNModel, foodRandomForest.FoodRandomForest],
[[
'nnmodels/nnmodel0.pkl', 'nnmodels/nnmodel1.pkl',
'nnmodels/nnmodel2.pkl', 'nnmodels/nnmodel3.pkl',
'nnmodels/nnmodel4.pkl', 'nnmodels/nnmodel5.pkl',
'nnmodels/nnmodel6.pkl', 'nnmodels/nnmodel7.pkl',
'nnmodels/nnmodel8.pkl', 'nnmodels/nnmodel9.pkl',
'nnmodels/nnmodel10.pkl', 'nnmodels/nnmodel11.pkl',
'nnmodels/nnmodel12.pkl', 'nnmodels/nnmodel13.pkl',
'nnmodels/nnmodel14.pkl', 'nnmodels/nnmodel15.pkl',
'nnmodels/nnmodel16.pkl', 'nnmodels/nnmodel17.pkl',
'nnmodels/nnmodel18.pkl', 'nnmodels/nnmodel19.pkl',
'nnmodels/nnmodel20.pkl', 'nnmodels/nnmodel21.pkl',
'nnmodels/nnmodel22.pkl', 'nnmodels/nnmodel23.pkl',
'nnmodels/nnmodel24.pkl', 'nnmodels/nnmodel25.pkl',
'nnmodels/nnmodel26.pkl', 'nnmodels/nnmodel27.pkl',
'nnmodels/nnmodel28.pkl', 'nnmodels/nnmodel29.pkl',
'nnmodels/nnmodel30.pkl'
],
[
'rf-models/randomforest0.joblib',
'rf-models/randomforest1.joblib',
'rf-models/randomforest2.joblib',
'rf-models/randomforest3.joblib',
'rf-models/randomforest4.joblib',
'rf-models/randomforest5.joblib',
'rf-models/randomforest6.joblib',
'rf-models/randomforest7.joblib',
'rf-models/randomforest8.joblib',
'rf-models/randomforest9.joblib',
'rf-models/randomforest10.joblib',
'rf-models/randomforest11.joblib',
'rf-models/randomforest12.joblib',
'rf-models/randomforest13.joblib',
'rf-models/randomforest14.joblib',
'rf-models/randomforest15.joblib',
'rf-models/randomforest16.joblib',
'rf-models/randomforest17.joblib',
'rf-models/randomforest18.joblib',
'rf-models/randomforest19.joblib',
'rf-models/randomforest20.joblib',
'rf-models/randomforest21.joblib',
'rf-models/randomforest22.joblib',
'rf-models/randomforest23.joblib',
'rf-models/randomforest24.joblib',
'rf-models/randomforest25.joblib',
'rf-models/randomforest26.joblib',
'rf-models/randomforest27.joblib',
'rf-models/randomforest28.joblib',
'rf-models/randomforest29.joblib'
]])
super(UltraProcessedFoodApp, self).__init__(**kwargs)
async def update_mocks():
directory = Path(__file__).parent
aggr = Aggregator(rapidapi_key=RAPIDAPI_KEY)
for source in aggr._sources:
source_name = type(source).__name__
file_path = directory / f'mock_response_{source_name}.txt'
response = await source.load_data()
with file_path.open('w') as file:
file.write(response)
class SimpleCentralizedGreedy:
def __init__(self):
self.aggregator = Aggregator() # to retrive server logs
self.server_set = set(util.retrieve_server_list()) # [server_ip, ]
self.content_set = set()
self.replica_map = {} # { file_uuid: servers that store the file }
self.replication_task = []
self.last_timestamp = 0 # the time stamp of last update
def update(self):
# update inner data
self.replica_map = {}
self.replication_task = []
# update content_set, replica_map
for server in self.server_set:
file_list = util.get_file_list_on_server(server)
for file_uuid in file_list:
self.content_set.add(file_uuid)
if file_uuid not in self.replica_map:
self.replica_map[file_uuid] = []
if server not in self.replica_map[file_uuid]:
self.replica_map[file_uuid].append(server)
current_timestamp = int(time.time())
logs = self.aggregator.get_redirect_log_entries(
self.last_timestamp, current_timestamp)
# used recently generated redirect logs to instruct replication
for log in logs:
timestamp, uuid, source, source_uuid, dest, req_type, status, response_size = log
dest = util.convert_to_local_hostname(dest)
if (uuid, dest) not in self.replication_task:
self.replication_task.append((uuid, dest))
self.last_timestamp = current_timestamp
def execute(self):
self.update()
for file_uuid, target in self.replication_task:
candidate_servers = self.replica_map[file_uuid]
target_simulation_ip = util.convert_to_simulation_ip(target)
source = util.find_closest_servers_with_ip(
target_simulation_ip, candidate_servers)[0]['server']
self.replicate(file_uuid, source, target)
def replicate(self, content, source, dest):
print 'Greedy: replicate file %s from %s to %s', (content, source,
dest)
if source == dest:
# a server can have at most one replcia, se we replicate to second nearest server
dest_simulation_ip = util.convert_to_simulation_ip(target)
candidate_servers = self.server_set - set(
self.replica_map[content])
dest = util.find_closest_servers_with_ip(
dest_simulation_ip, candidate_servers)[0]['server']
util.replicate(content, source, dest)
class Evaluator:
def __init__(self, start_time, end_time):
self.set_time(start_time, end_time)
self.aggregator = Aggregator()
def set_time(self, start_time, end_time):
self.start_time = start_time
self.end_time = end_time
# read_logs and moving_logs are always emited unless for testing
def evaluate(self, read_logs=None, moving_logs=None):
# collect server logs
if read_logs is None:
read_logs = self.aggregator.get_read_log_entries(
self.start_time, self.end_time)
if moving_logs is None:
moving_logs = self.aggregator.get_moving_log_entries(
self.start_time, self.end_time)
# calculate the average latency
latency_sum = 0
request_count = 0
ip_cache = ip_location_cache()
for log in read_logs:
timestamp, uuid, source, source_uuid, dest, req_type, status, response_size = log
client_loc = ip_cache.get_lat_lon_from_ip(source)
server_loc = ip_cache.get_lat_lon_from_ip(dest)
distance = util.get_distance(client_loc, server_loc)
unit = 1000.0
latency = distance / unit
request_importance = 1
latency_sum += latency * request_importance
request_count += request_importance
average_latency = latency_sum / request_count
inter_datacenter_traffic = 0
for log in moving_logs:
timestamp, uuid, source, source_uuid, dest, req_type, status, response_size = log
# treat all files as uniform size
inter_datacenter_traffic += 1
# display latency, cost, etc
return average_latency, inter_datacenter_traffic
class Validator(object):
def __init__(self, data, classifier):
self.data = data
self.classifier = classifier
self.aggregator = Aggregator()
def validate(self, test_frame=0.1):
frame_l = math.floor(len(self.data) * test_frame)
last = 0
while last + frame_l < len(self.data):
# print(str(last) + "/" + str(len(self.data)))
last += frame_l
test_data = self.data[last - frame_l:last]
train_data = pd.DataFrame().append(self.data[:last - frame_l]). \
append(self.data[last:])
self.classifier.train(train_data)
for index, example in test_data.iterrows():
p_result = self.classifier.predict(example)
# print(index)
self.aggregator.count(example, p_result)
self.aggregator.next()
def run():
ctx = zmq.asyncio.Context()
loop = zmq.asyncio.ZMQEventLoop()
asyncio.set_event_loop(loop)
socket = ctx.socket(zmq.PULL)
pool = init_db_pool(loop=loop)
logger.debug('Connected to database')
socket.bind(ZMQ_ADDRESS)
logger.info(f'Listening to {ZMQ_ADDRESS} ...')
aggregator = Aggregator(zmq_socket=socket, db_pool=pool)
try:
loop.run_until_complete(aggregator.run())
except KeyboardInterrupt:
logger.info('Shutting down ...')
finally:
socket.disconnect()
loop.run_until_complete(pool.close())
logger.info('Disconnected from ZMQ socket')
def __init__(self, line, debug):
reader = EffectReader(line)
# reader = TraceReader(line)
reader.parse_trace()
self.code_size = len(reader.code)
signal.signal(signal.SIGALRM, handler)
signal.alarm(15)
# print(reader.signature)
optimizer = Aggregator(reader.code)
ExpressionExecutor(reader, optimizer, debug)
signal.alarm(0)
async def test_update():
aggr = Aggregator()
await aggr.update()
aggr._rapidapi.update.assert_called_once()
aggr._github.update.assert_called_once()
aggr._stopcorona.update.assert_called_once()
with patch.object(RapidapiSource, 'update', _async_raise(ReadTimeout)):
aggr = Aggregator()
await aggr.update()
logging.error.assert_called_with('ReadTimeout in source update')
with patch.object(GithubSource, 'update', _async_raise(ConnectTimeout)):
aggr = Aggregator()
await aggr.update()
logging.error.assert_called_with('ConnectTimeout in source update')
with patch.object(GithubSource, 'update', _async_raise(ZeroDivisionError)):
aggr = Aggregator()
await aggr.update()
logging.exception.assert_called_once()
def main() -> None:
args_parser = argparse.ArgumentParser()
args_parser.add_argument("--cache-dir",
dest="cache_dir",
metavar="DIR",
type=str,
default=".cache")
args_parser.add_argument("--export-dir",
dest="export_dir",
metavar="DIR",
type=str,
default=".export")
args_parser.add_argument(
"--templates-dir",
dest="templates_dir",
metavar="DIR",
type=str,
default="templates",
)
args_parser.add_argument("--base-url",
dest="base_url",
metavar="URL",
type=str)
args_parser.add_argument(
"--podcasts-json",
dest="podcasts_json",
metavar="FILE",
type=str,
default="podcasts.json",
)
args_parser.add_argument("--clear-cache",
dest="clear_cache",
action="store_true")
args_parser.add_argument("--keep-feeds",
dest="keep_feeds",
action="store_true")
args = args_parser.parse_args()
a = Aggregator(
podcasts_json=args.podcasts_json,
cache_dir=args.cache_dir,
export_dir=args.export_dir,
templates_dir=args.templates_dir,
base_url=args.base_url if args.base_url else "file://{}".format(
os.path.abspath(args.export_dir)),
)
if args.clear_cache:
a.clear_cache()
a.sync(keep_feeds=args.keep_feeds)
a.export()
def main():
nasdaq=StockMarket("NASDAQ", ["AAPL", "CSCO", "MSFT", "GOOG"])
newyork=StockMarket("NYSE", ["IBM", "HPQ", "BP"])
agg=Aggregator()
agg.add_symbols(nasdaq.symbols())
agg.add_symbols(newyork.symbols())
print("aggregated symbols:", agg.available_symbols())
nasdaq.listener(agg)
newyork.listener(agg)
view=Viewer()
agg.view(view, ["IBM", "AAPL", "MSFT"])
print("")
while True:
nasdaq.generate()
newyork.generate()
time.sleep(0.5)
def go(b1, b2, b3, p, output):
"""
Launches the triangulation for the given input file and writes result in new CSV
:return:
"""
tab = Aggregator().aggregate(b1, b2, b3, p)
with open(output, 'w') as csvfile:
csv_writer = csv.writer(csvfile,
delimiter=',',
quotechar='|',
quoting=csv.QUOTE_MINIMAL)
for row in tab:
csv_writer.writerow(row)
return 0
def __init__(self, args, n_entity, n_relation,adj_entity,adj_relation): # 初始化使用 args n_entity n_relation
super(RippleNet, self).__init__()
self._parse_args(args, n_entity, n_relation)
self.entity_emb = nn.Embedding(self.n_entity, self.dim)
self.relation_emb = nn.Embedding(self.n_relation, self.dim * self.dim)
self.relation_emb_GCN = nn.Embedding(self.n_relation,self.dim)
self.transform_matrix = nn.Linear(self.dim, self.dim, bias=False)
self.criterion = nn.BCELoss()
self.transformerEncoderLayer = nn.TransformerEncoderLayer(d_model=self.dim, nhead=self.n_head, dim_feedforward=self.feed_f_dim)
self.transformerEncoder = nn.TransformerEncoder(num_layers=1, encoder_layer=self.transformerEncoderLayer)
self.pooling = nn.AvgPool2d
self.linear = nn.Linear(in_features=2 * self.dim, out_features=self.dim)
self.adj_entity = adj_entity
self.adj_relation = adj_relation
self.aggregator = Aggregator(self.batch_size, self.dim, args.aggregator)
def main():
nasdaq=StockMarket('NASDAQ',['AAPL','CSCO','MSFT','GOOG'])
newyork=StockMarket('NYSE',['IBM','HPQ','BP'])
agg = Aggregator()
agg.add_symbols(nasdaq.symbols())
agg.add_symbols(newyork.symbols())
print("aggregated symbols: ", agg.available_symbols())
nasdaq.listener(agg)
newyork.listener(agg)
view=Viewer()
agg.view(view, ['IBM','AAPL','MSFT'])
print("")
while True:
nasdaq.generate()
newyork.generate()
time.sleep(0.5)
def __init__(self, num_user, num_ent, num_rel, kg, args, device):
super(KGCN, self).__init__()
self.num_user = num_user
self.num_ent = num_ent
self.num_rel = num_rel
self.n_iter = args.n_iter
self.batch_size = args.batch_size
self.dim = args.dim
self.n_neighbor = args.neighbor_sample_size
self.kg = kg
self.device = device
self.aggregator = Aggregator(self.batch_size, self.dim,
args.aggregator)
self._gen_adj()
self.usr = torch.nn.Embedding(num_user, args.dim)
self.ent = torch.nn.Embedding(num_ent, args.dim)
self.rel = torch.nn.Embedding(num_rel, args.dim)
def main():
# capture the config path from the run arguments
# then process the json configuration file
try:
args = get_args()
config = process_config(args.config)
except:
print("missing or invalid arguments")
exit(0)
# create the experiments dirs
create_dirs([config['result_dir'], config['checkpoint_dir'], config['checkpoint_dir_lstm']])
# save the config in a txt file
save_config(config)
# create tensorflow session
sessions = []
data = []
model_vaes = []
vae_trainers = []
lstm_models = []
model_vae_global = VAEmodel(config, "Global")
sess_global = tf.Session(config=tf.ConfigProto())
for i in range(1, 10):
sess = tf.Session(config=tf.ConfigProto())
sessions.append(sess)
data.append(generator_fl(config, i))
model_vaes.append(VAEmodel(config, "Client{}".format(i)))
model_vaes[-1].load(sessions[-1])
vae_trainers.append(vaeTrainer(sessions[-1], model_vaes[-1], data[-1], config))
lstm_models.append(lstmKerasModel("Client{}".format(i), config))
model_vae_global.load(sess_global)
trainer_vae_global = vaeTrainer(sess_global, model_vae_global, data[0], config)
lstm_model_global = lstmKerasModel("Global", config)
client_weights = [0.1] * 8
client_weights.append(0.2)
aggregator = Aggregator(vae_trainers, trainer_vae_global, lstm_models, lstm_model_global, config, client_weights)
aggregator.aggregate_vae()
aggregator.aggregate_lstm()
class BlockCoder:
def __init__(self, code_f, needed_size, can_be_flushed):
self.code_fun = code_f
self.needed_size = needed_size
self.can_be_flushed = can_be_flushed
self.aggregator = Aggregator()
def process(self, data: memoryview) -> memoryview:
self.aggregator.push(data)
result = bytearray()
while self.aggregator.has_pull_data(self.needed_size):
d = self.aggregator.pull(self.needed_size)
result.extend(self.code_fun(d))
return memoryview(result)
def flush(self) -> memoryview:
if self.aggregator.total_data_size() == 0:
return bytes()
if self.can_be_flushed:
result = self.code_fun(self.aggregator.flush())
return memoryview(result)
else:
raise IOError()
def test_check_operator(self):
"""
This test is not working because the Exception is not caught by the context : TO BE CORRECTED
:return: None
"""
with self.assertRaises(ValueError): Aggregator(self.comp_parent, [self.comp_1, self.comp_3], 'Invalid_Code')
def test_div(self):
agg = Aggregator(self.comp_parent, [self.comp_1, self.comp_3], Operator.DIV)
agg.aggregate()
self.assertEqual(self.comp_parent.data.equals(pd.Series([0.5, 1, 1.5, 2, 2.5], index=self.idx_1)), True)
def test_find_near():
db = DB()
aggregator = Aggregator(db)
car_id_1 = aggregator.login_car()
car_id_2 = aggregator.login_car()
car_id_3 = aggregator.login_car()
car_1 = db.get_car(car_id_1)
car_2 = db.get_car(car_id_2)
car_3 = db.get_car(car_id_3)
aggregator.new_position_car(car_id_1, 10, 10)
aggregator.new_position_car(car_id_2, 20, 20)
aggregator.new_position_car(car_id_3, 30, 30)
assert set(db.find_near(Point(5, 5), 2)) == set([car_1, car_2])
aggregator.logout_car(car_id_1, "Test reason")
aggregator.logout_car(car_id_2, "Test reason")
aggregator.logout_car(car_id_3, "Test reason")
with pytest.raises(CarNotFound):
db.get_car(car_id_1)
with pytest.raises(CarNotFound):
db.get_car(car_id_2)
with pytest.raises(CarNotFound):
db.get_car(car_id_3)
from aggregator import Aggregator from relations import Relator from stager import Stager from social import Social from mf import ImplicitMF from top_aggregator import TopAggregator print print '######### Aggregating media #########' print aggregator = Aggregator() aggregator.process() print print '######### Grabbing Top Media #########' print top_aggregator = TopAggregator() top_aggregator.process() print print '######### Forming relations via tag analysis #########' print relator = Relator() relator.process() print print '######### Running Implicit Matrix Factorization ########' print mf = ImplicitMF() mf.process() print print '######### Running Social Recommendations #########' print social = Social()
if not args.field:
args.field = ["follow/day/vendor",
"favorite/day/vendor",
"favorite/day/product",
"total/day/vendor",
"follow/vendor",
"total/product"
]
args.field = [Field(f) for f in args.field]
return args
def convert_timestamp(dict_):
if "time" in dict_:
dict_["time"] = datetime.fromtimestamp(dict_["time"])
return dict_
def iterate_records(filename, line_parser=json.loads):
for line in open(filename):
yield line_parser(line)
if __name__ == "__main__":
args = parse_args()
aggregator = Aggregator(args.field)
line_parser = funcutils.and_then(json.loads, convert_timestamp)
for r in iterate_records(args.filename, line_parser):
aggregator.add(r)
aggregator.print_stats()
