def find_user(username, domain_name): ''' Returns user id (str). Parameters: username (str): user name. domain_name (str): domain name. Returns: user_id (str): User id. In case of error: Returns -1 (int). ''' url = f'https://{domain_name}/api/v1/accounts/{username}' json_user, status_code = shared.fetch(url) json_user = json_user.text if json_user == -1: return -1 dict_data = shared.load_json(json_user) if not isinstance(dict_data, dict): sys.stderr.write('Error parsing JSON data.\n') return -1 if 'id' in dict_data: user_id = dict_data['id'] return user_id else: sys.stderr.write('No user id.\n') return -1
def radio_play(play_status, json_dir, audio_dir): global schedule_dict if play_status == False: # list_programs(schedule_dict) year, month, day = sh.set_date() logging.debug("Current date set") raw_schedule_dict = sh.load_json(year, month, day, json_dir) logging.debug("JSON imported") schedule_dict = sh.convert_dict_dates(raw_schedule_dict) logging.debug("Dict times converted: %s records", str(len(schedule_dict))) play_file_index, play_file = find_audio_file(schedule_dict) logging.debug("Found file to play: %s", play_file) start_time_str = find_start_time(schedule_dict, play_file_index) logging.debug("Found start time: %s", start_time_str) popen = subprocess.Popen([ 'omxplayer', '-o', 'local', audio_dir + play_file, '--pos=' + start_time_str ], stdout=subprocess.PIPE, universal_newlines=True) play_status = True logging.info("Started playing") time.sleep(0.25) return play_status else: subprocess.call(['killall', 'omxplayer.bin']) play_status = False logging.info("Stopped playing") time.sleep(0.25) return play_status
def scrape(url, domain_name, user_id): ''' Returns list of posts (list) and URL for the next url to scrape (str) as a tuple. Parameters: url (str): API URL. domain_name (str): domain name. user_id (str): User id. Returns: lst_out (list), url (str): List of posts and next url. In case of error: Returns -1, -1 (integers). ''' # Example URLs: # See the most recent posts. # https://{domain_name}/api/v1/accounts/{user_id}/statuses?with_muted=true&limit=40&exclude_reblogs=true # {max_id}: Only posts older than this post's id will be shown. # https://{domain_name}/api/v1/accounts/{user_id}/statuses?max_id={max_id}&with_muted=true&limit=40&exclude_reblogs=true lst_posts = [] lst_out = [] json_posts, status_code = shared.fetch(url) if json_posts == -1: return -1, -1 json_posts = json_posts.text lst_posts = shared.load_json(json_posts) if not isinstance(lst_posts, list): sys.stderr.write('Broken list of posts.\n') return -1, -1 if not lst_posts: url = '' return lst_posts, url for post in lst_posts: if isinstance(post, dict): dict_post = {} if 'created_at' in post.keys(): dict_post['datetime'] = post['created_at'] if 'content' in post.keys(): dict_post['content'] = post['content'] if 'url' in post.keys(): dict_post['url'] = post['url'] lst_out.append(dict_post) # Use the id of the last post to keep fetching older posts. last_post = lst_posts[-1] if 'id' in last_post.keys(): max_id = last_post['id'] url = f'https://{domain_name}/api/v1/accounts/{user_id}/statuses?max_id={max_id}&with_muted=true&limit=40&exclude_reblogs=true' return lst_out, url
def get_tasks(files): gateway_count = 1 # make the same "random" choices every time shared.seed_random(12356) tasks = [] for path in sorted(glob.glob(files)): state = shared.load_json(path) tasks.append((path, ping.get_random_nodes(state, gateway_count))) return tasks
def scrape(domain_name, payload): ''' Returns list of posts (list) and post id for the next posts to fetch (str) as a tuple. Parameters: domain_name (str): domain name. payload (dict): Request payload Returns: lst_out (list), untilId (str): List of posts and id of the last post. In case of error: Returns -1, -1 (integers). ''' url = f'https://{domain_name}/api/users/notes' lst_posts = [] lst_out = [] untilId = '' json_posts, status_code = shared.fetch(url, http_method='POST', payload=payload) if json_posts == -1: return -1, -1 json_posts = json_posts.text lst_posts = shared.load_json(json_posts) if not isinstance(lst_posts, list): sys.stderr.write('Broken list of posts.\n') return -1, -1 if not lst_posts: url = '' return lst_posts, untilId for post in lst_posts: if isinstance(post, dict): dict_post = {} if 'createdAt' in post.keys(): dict_post['datetime'] = post['createdAt'] if 'text' in post.keys(): dict_post['content'] = post['text'] if 'id' in post.keys(): dict_post['url'] = f'https://{domain_name}/notes/{post["id"]}' lst_out.append(dict_post) # Use the id of the last post to keep fetching older posts. last_post = lst_posts[-1] if 'id' in last_post.keys(): untilId = last_post['id'] return lst_out, untilId
def run(protocol, files, csvfile): shared.seed_random(1234) for path in sorted(glob.glob(files)): state = shared.load_json(path) (node_count, link_count) = shared.json_count(state) print(f'run {protocol} on {path}') network.apply(state=state, link_command=get_tc_command, remotes=remotes) shared.sleep(10) for offset in range(0, 60, 2): tmp_ms = shared.millis() traffic_beg = traffic.traffic(remotes) traffic_ms = shared.millis() - tmp_ms tmp_ms = shared.millis() software.start(protocol) software_ms = shared.millis() - tmp_ms # Wait until wait seconds are over, else error shared.sleep(offset) paths = ping.get_random_paths(state, 2 * 200) paths = ping.filter_paths(state, paths, min_hops=2, path_count=200) ping_result = ping.ping(paths=paths, duration_ms=2000, verbosity='verbose', remotes=remotes) traffic_end = traffic.traffic(remotes) sysload_result = shared.sysload(remotes) software.clear(remotes) # add data to csv file extra = (['node_count', 'traffic_ms', 'software_ms', 'offset_ms'], [node_count, traffic_ms, software_ms, offset * 1000]) shared.csv_update(csvfile, '\t', extra, (traffic_end - traffic_beg).getData(), ping_result.getData(), sysload_result) network.clear(remotes)
def run(protocol, files, csvfile): for path in sorted(glob.glob(files)): state = shared.load_json(path) (node_count, link_count) = shared.json_count(state) # Limit node count to 300 if node_count > 300: continue print(f'run {protocol} on {path}') network.apply(state=state, link_command=get_tc_command, remotes=remotes) shared.sleep(10) software_start_ms = shared.millis() software.start(protocol, remotes) software_startup_ms = shared.millis() - software_start_ms shared.sleep(300) start_ms = shared.millis() traffic_beg = traffic.traffic(remotes) paths = ping.get_random_paths(state, 2 * 200) paths = ping.filter_paths(state, paths, min_hops=2, path_count=200) ping_result = ping.ping(remotes=remotes, paths=paths, duration_ms=300000, verbosity='verbose') traffic_ms = shared.millis() - start_ms traffic_end = traffic.traffic(remotes) sysload_result = shared.sysload(remotes) software.clear(remotes) network.clear(remotes) # add data to csv file extra = (['node_count', 'traffic_ms', 'software_startup_ms'], [node_count, traffic_ms, software_startup_ms]) shared.csv_update(csvfile, '\t', extra, (traffic_end - traffic_beg).getData(), ping_result.getData(), sysload_result)
def download_metadata(self, n_threads=5): web_request_queue = Queue() json_serialization_queue = Queue() urls = self.paginated_media_urls if len(urls) > 1: for url in urls: web_request_queue.put(url) web_thread = lambda: ThreadMetadataRequest( web_request_queue, json_serialization_queue, self.session ) pool_size = min(len(urls), n_threads) web_pool = [web_thread() for x in range(pool_size)] json_serializer = ThreadJSONWriter( json_serialization_queue, self.metadata_filepath ) for thread in web_pool: thread.setDaemon(True) thread.start() json_serializer.start() web_request_queue.join() json_serialization_queue.join() else: json_response = self._grab_json(urls[0]) media_entries = json_response['media'] media_dict = list_of_dicts_to_dict( media_entries, promote_to_key='_id') exists = osp.isfile(self.metadata_filepath) filemode = 'r+w' if exists else 'w' with open(self.metadata_filepath, filemode) as f: try: cached_meta = load_json(f) if exists else {} except ValueError: cached_meta = {} cached_meta.update(media_dict) dump_json(cached_meta, f) self._metadata = cached_meta
def downloader(hours_ahead, audio_dir, json_dir): year, month, day = sh.set_date() logging.debug('Current date set') rec_start_time, rec_end_time = get_record_times(year, month, day, hours_ahead) logging.debug('Got record times') raw_schedule_dict = sh.load_json(year, month, day, json_dir) logging.debug('JSON imported') schedule_dict = sh.convert_dict_dates(raw_schedule_dict) logging.debug('Dict times converted: %s records', str(len(schedule_dict))) download_list = get_download_list(schedule_dict, rec_start_time, rec_end_time, audio_dir) logging.info('Download list compiled') init_download(download_list, audio_dir) logging.info('Downloads completed') # TO DO: Check success of downloads return
def run(protocol, csvfile): for path in sorted(glob.glob(f'../../data/grid4/*.json')): state = shared.load_json(path) (node_count, link_count) = shared.json_count(state) print(f'run {protocol} on {path}') network.apply(state=state, link_command=get_tc_command, remotes=remotes) shared.sleep(10) software_start_ms = shared.millis() software.start(protocol, remotes) software_startup_ms = shared.millis() - software_start_ms shared.sleep(30) paths = ping.get_random_paths(state, 2 * link_count) paths = ping.filter_paths(state, paths, min_hops=2, path_count=link_count) ping_result = ping.ping(remotes=remotes, paths=paths, duration_ms=30000, verbosity='verbose') sysload_result = shared.sysload(remotes) software.clear(remotes) # add data to csv file extra = (['node_count', 'software_startup_ms'], [node_count, software_startup_ms]) shared.csv_update(csvfile, '\t', extra, ping_result.getData(), sysload_result) network.clear(remotes) # abort benchmark when less then 40% of the pings arrive if ping_result.transmitted == 0 or (ping_result.received / ping_result.transmitted) < 0.4: break
def run(protocol, csvfile): shared.seed_random(1377) for path in sorted(glob.glob(f'../../data/freifunk/*.json')): state = shared.load_json(path) (node_count, link_count) = shared.json_count(state) dataset_name = '{}-{:04d}'.format(os.path.basename(path)[9:-5], node_count) # limit to what the host can handle if node_count > 310: continue print(f'run {protocol} on {path}') state = network.apply(state=state, link_command=get_tc_command, remotes=remotes) shared.sleep(10) software.start(protocol, remotes) shared.sleep(300) start_ms = shared.millis() traffic_beg = traffic.traffic(remotes) paths = ping.get_random_paths(state, 2 * node_count) paths = shared.filter_paths(state, paths, min_hops=2, path_count=node_count) ping_result = shared.ping(remotes=remotes, paths=paths, duration_ms=300000, verbosity='verbose') sysload_result = shared.sysload(remotes) traffic_ms = shared.millis() - start_ms traffic_end = traffic.traffic(remotes) software.clear(remotes) # add data to csv file extra = (['dataset_name', 'node_count', 'traffic_ms'], [dataset_name, node_count, traffic_ms]) shared.csv_update(csvfile, '\t', extra, (traffic_end - traffic_beg).getData(), ping_result.getData(), sysload_result) network.clear(remotes)
def run(protocol, tasks, csvfile): for path, gateways in tasks: state = shared.load_json(path) (node_count, link_count) = shared.json_count(state) # Limit node count to 300 if node_count > 300: continue print(f'run {protocol} on {path}') network.apply(state=state, remotes=remotes) shared.sleep(10) software_start_ms = shared.millis() software.start(protocol, remotes) software_startup_ms = shared.millis() - software_start_ms shared.sleep(30) start_ms = shared.millis() traffic_beg = traffic.traffic(remotes) paths = ping.get_paths_to_gateways(state, gateways) ping_result = ping.ping(remotes=remotes, paths=paths, duration_ms=300000, verbosity='verbose') traffic_ms = shared.millis() - start_ms traffic_end = traffic.traffic(remotes) sysload_result = shared.sysload(remotes) software.clear(remotes) network.clear(remotes) # add data to csv file extra = (['node_count', 'traffic_ms', 'software_startup_ms'], [node_count, traffic_ms, software_startup_ms]) shared.csv_update(csvfile, '\t', extra, (traffic_end - traffic_beg).getData(), ping_result.getData(), sysload_result)
#!/usr/bin/python3 from shared import load_json, plot_many, get_v_function_arg v_functions = get_v_function_arg() if v_functions == "small": h1 = load_json("networks/legacy.training.json") h2 = load_json("networks/n_tuple_small.training.json") plot_many([("Legacy", h1), ("N-Tuple Small", h2)]) elif v_functions == "medium": h = load_json("networks/n_tuple_medium.training.json") plot_many([("N-Tuple Medium", h)])
def deserialize_metadata(self, return_iterator=False): if self._metadata_exists(): with open(self.metadata_filepath, 'r') as f: metadata = load_json(f) return metadata
(traffic_end - traffic_begin).getData(), ping_result.getData(), sysload_result) return (100.0 * ping_result.received / ping_result.send) # Keep track of tests that exceed the machines resources and skip bigger networks drop_test = set() protocols = [ 'babel', 'batman-adv', 'bmx6', 'bmx7', 'cjdns', 'olsr1', 'olsr2', 'ospf', 'yggdrasil' ] for topology in ['line', 'grid4', 'grid8', 'rtree']: for path in sorted(glob.glob(f'../../data/{topology}/*.json')): state = shared.load_json(path) (node_count, link_count) = shared.json_count(state) # No test to be done for this topology if all((f'{p}_{topology}' in drop_test) for p in protocols): continue # Create network network.apply(state=state, remotes=remotes) for protocol in protocols: if f'{protocol}_{topology}' in drop_test: continue pc = run(topology, path, state)