def run(protocol, csvfile):
for path in sorted(glob.glob(f'../../data/grid4/*.json')):
state = tools.load_json(path)
(node_count, link_count) = tools.json_count(state)
print(f'run {protocol} on {path}')
network.apply(state=state, link_command=get_tc_command, remotes=remotes)
tools.sleep(10)
software_start_ms = tools.millis()
software.start(protocol, remotes)
software_startup_ms = tools.millis() - software_start_ms
tools.sleep(30)
paths = tools.get_random_paths(state, 2 * link_count)
paths = tools.filter_paths(state, paths, min_hops=2, path_count=link_count)
ping_result = tools.ping_paths(remotes=remotes, paths=paths, duration_ms=30000, verbosity='verbose')
sysload_result = tools.sysload(remotes)
software.clear(remotes)
# add data to csv file
extra = (['node_count', 'software_startup_ms'], [node_count, software_startup_ms])
tools.csv_update(csvfile, '\t', extra, ping_result.getData(), sysload_result)
network.clear(remotes)
# abort benchmark when less then 40% of the pings arrive
if (ping_result.received / ping_result.transmitted) < 0.4:
break
def reset(self, start_at):
if start_at > 0:
if self.json_path is not None:
if self.json_path.exists():
self.H = load_json(self.json_path)
for k in self.H.keys():
self.H[k] = self.H[k][:start_at]
def __init__(self,
task,
max_len=None,
loss_weight=0.25,
regularizers=0.001,
lstm_in_dropout=0.5,
lstm_dropout=0.5,
fc_dropout=0.5):
source_dir = os.path.join('.', 'dataset', 'data', task)
if Config.word_vocab is None:
Config.word_vocab, _ = load_vocab(
os.path.join(source_dir, '..', 'words.vocab'))
Config.vocab_size = len(Config.word_vocab)
Config.word_emb = load_embeddings(
os.path.join(source_dir, '..', 'glove.filtered.npz'))
Config.emb_len = len(Config.word_emb[0])
Config.word_num = len(Config.word_emb)
self.max_len = max_len
self.loss_weight = loss_weight
self.regularizers = regularizers
self.lstm_in_dropout = lstm_in_dropout
self.lstm_dropout = lstm_dropout
self.fc_dropout = fc_dropout
self.label_size = load_json(os.path.join(source_dir,
'label.json'))["label_size"]
self.load_data(source_dir)
def test_update_json1(self):
data1 = {'a': 1, 'b': 2}
data2 = {'c': 3, 'd': 4}
data3 = {'a': 1, 'c': 3, 'b': 2, 'd': 4}
output_file = os.path.join(fixture_dir,
'foo_{0}.json'.format(t.timestamp()))
t.write_json(object=data1, output_file=output_file)
t.update_json(data=data2, input_file=output_file)
data4 = t.load_json(input_file=output_file)
self.assertTrue(
data3 == data4,
'Data read from JSON file does not match expected output')
def make_coin_objects(external):
coin_info = load_json('coin_info.json')
coins = []
for coin in coin_info.keys():
#Fetch the abi if key isn't prevent in file
#Allows for adding coins without looking for abi
coin_info[coin]['abi'] = coin_info[coin].get('abi', get_abi(coin['address']))
coins.append(Coin(info['address'], coin_info[coin]['abi'], external))
with open('coin_info.json',w) as f:
json.dump(coin_info, f)
return coins
def get_train_test_from_file(self, path):
data = load_json(path)
x = list()
y = list()
for line in data:
x.append(line['sentence'])
y.append(line['label'])
if self.max_len is None:
self.max_len = max(map(lambda x: len(x), x))
x = sequence.pad_sequences(x,
maxlen=self.max_len,
padding='post',
truncating='post')
return x, y
def run(protocol, files, csvfile):
tools.seed_random(1234)
for path in sorted(glob.glob(files)):
state = tools.load_json(path)
(node_count, link_count) = tools.json_count(state)
print(f'run {protocol} on {path}')
network.apply(state=state, link_command=get_tc_command)
tools.sleep(10)
for offset in range(0, 60, 2):
tmp_ms = tools.millis()
traffic_beg = tools.traffic()
traffic_ms = tools.millis() - tmp_ms
tmp_ms = tools.millis()
software.start(protocol)
software_ms = tools.millis() - tmp_ms
# Wait until wait seconds are over, else error
tools.sleep(offset)
paths = tools.get_random_paths(state, 2 * 200)
paths = tools.filter_paths(state,
paths,
min_hops=2,
path_count=200)
ping_result = tools.ping_paths(paths=paths,
duration_ms=2000,
verbosity='verbose')
traffic_end = tools.traffic()
sysload_result = tools.sysload()
software.clear()
# add data to csv file
extra = (['node_count', 'traffic_ms', 'software_ms', 'offset_ms'],
[node_count, traffic_ms, software_ms, offset * 1000])
tools.csv_update(csvfile, '\t', extra,
(traffic_end - traffic_beg).getData(),
ping_result.getData(), sysload_result)
network.clear()
def __init__(self, json_labels=None, data_dir=None, group=None):
self._log = logging.getLogger('root')
self._log.info(f'Loading labels from {json_labels} for group: {group}')
self._labels = tools.load_json(json_labels)
self._group = group
self._data_dir = tools.str2path(data_dir)
self._labels = self._filter_labels()
self._group_labels = self._get_group_labels()
self._numerical_labels, self._labels2num = self._get_numerical_labels()
self._labels_counts = self._get_labels_counts()
self._one_hot_labels = to_categorical(self._numerical_labels)
self._total_count = len(self._group_labels)
self._inverse_frequency = self._get_inverse_frequency()
self._proportions = self._get_proportions()
self._image2one_hot = self._get_image2one_hot()
def run(protocol, files, csvfile):
for path in sorted(glob.glob(files)):
state = tools.load_json(path)
(node_count, link_count) = tools.json_count(state)
# Limit node count to 300
if node_count > 300:
continue
print(f'run {protocol} on {path}')
network.apply(state=state,
link_command=get_tc_command,
remotes=remotes)
tools.sleep(10)
software_start_ms = tools.millis()
software.start(protocol, remotes)
software_startup_ms = tools.millis() - software_start_ms
tools.sleep(300)
start_ms = tools.millis()
traffic_beg = tools.traffic(remotes)
paths = tools.get_random_paths(state, 2 * 200)
paths = tools.filter_paths(state, paths, min_hops=2, path_count=200)
ping_result = tools.ping_paths(remotes=remotes,
paths=paths,
duration_ms=300000,
verbosity='verbose')
traffic_ms = tools.millis() - start_ms
traffic_end = tools.traffic(remotes)
sysload_result = tools.sysload(remotes)
software.clear(remotes)
network.clear(remotes)
# add data to csv file
extra = (['node_count', 'traffic_ms', 'software_startup_ms'],
[node_count, traffic_ms, software_startup_ms])
tools.csv_update(csvfile, '\t', extra,
(traffic_end - traffic_beg).getData(),
ping_result.getData(), sysload_result)
def run(protocol, csvfile):
tools.seed_random(1377)
for path in sorted(glob.glob(f'../../data/freifunk/*.json')):
state = tools.load_json(path)
(node_count, link_count) = tools.json_count(state)
dataset_name = '{}-{:04d}'.format(os.path.basename(path)[9:-5], node_count)
# limit to what the host can handle
if node_count > 310:
continue
print(f'run {protocol} on {path}')
state = network.apply(remotes=remotes, state=state, link_command=get_tc_command)
tools.sleep(10)
software.start(protocol, remotes)
tools.sleep(300)
start_ms = tools.millis()
traffic_beg = tools.traffic(remotes)
paths = tools.get_random_paths(state, 2 * node_count)
paths = tools.filter_paths(state, paths, min_hops=2, path_count=node_count)
ping_result = tools.ping_paths(remotes=remotes, paths=paths, duration_ms=300000, verbosity='verbose')
sysload_result = tools.sysload(remotes)
traffic_ms = tools.millis() - start_ms
traffic_end = tools.traffic(remotes)
software.clear(remotes)
# add data to csv file
extra = (['dataset_name', 'node_count', 'traffic_ms'], [dataset_name, node_count, traffic_ms])
tools.csv_update(csvfile, '\t', extra, (traffic_end - traffic_beg).getData(), ping_result.getData(), sysload_result)
network.clear(remotes)
def build_bar():
icon_theme = Gtk.IconTheme.get_default()
orientation = Gtk.Orientation.VERTICAL if args.vertical else Gtk.Orientation.HORIZONTAL
box = Gtk.Box(orientation=orientation)
appendix = load_json(build_from_file)
for entry in appendix:
name = entry["name"]
exec = entry["exec"]
icon = entry["icon"]
image = None
if icon.startswith('/'):
try:
pixbuf = GdkPixbuf.Pixbuf.new_from_file_at_size(icon, args.s, args.s)
image = Gtk.Image.new_from_pixbuf(pixbuf)
except:
pass
else:
try:
if icon.endswith('.svg') or icon.endswith('.png'):
icon = entry.icon.split('.')[0]
pixbuf = icon_theme.load_icon(icon, args.s, Gtk.IconLookupFlags.FORCE_SIZE)
image = Gtk.Image.new_from_pixbuf(pixbuf)
except:
pass
button = Gtk.Button()
button.set_property("name", "button")
button.set_always_show_image(True)
button.set_image(image)
button.set_image_position(Gtk.PositionType.TOP)
button.set_label(name)
button.set_property("width_request", args.bw)
button.set_property("height_request", args.bh)
button.connect('clicked', launch, exec)
box.pack_start(button, False, False, int(args.p / 2))
return box
def build_menu():
icon_theme = Gtk.IconTheme.get_default()
menu = Gtk.Menu()
if not args.no_menu:
win.search_item = Gtk.MenuItem()
win.search_item.add(win.search_box)
win.search_item.set_sensitive(False)
menu.add(win.search_item)
# Prepend favourite items (-f or -fn argument used)
favs_number = 0
if args.favourites:
favs_number = 5
elif args.fn:
favs_number = args.fn
if favs_number > 0:
global sorted_cache
if len(sorted_cache) < favs_number:
favs_number = len(sorted_cache)
to_prepend = [] # list of favourite items
for i in range(favs_number):
fav_exec = sorted_cache[i][0]
for item in all_entries:
if item.exec == fav_exec and item not in to_prepend:
to_prepend.append(item)
break # stop searching, there may be duplicates on the list
# build menu items
for entry in to_prepend:
name = entry.name
exec = entry.exec
icon = entry.icon
hbox = Gtk.HBox()
label = Gtk.Label()
label.set_text(name)
image = None
if icon.startswith('/'):
try:
pixbuf = GdkPixbuf.Pixbuf.new_from_file_at_size(
icon, args.s, args.s)
image = Gtk.Image.new_from_pixbuf(pixbuf)
except:
pass
else:
try:
if icon.endswith('.svg') or icon.endswith('.png'):
icon = entry.icon.split('.')[0]
pixbuf = icon_theme.load_icon(
icon, args.s, Gtk.IconLookupFlags.FORCE_SIZE)
image = Gtk.Image.new_from_pixbuf(pixbuf)
except:
pass
if image:
hbox.pack_start(image, False, False, 10)
if name:
hbox.pack_start(label, False, False, 0)
item = Gtk.MenuItem()
item.set_property("name", "item-favorites")
item.add(hbox)
item.connect('activate', launch, exec)
menu.append(item)
if to_prepend:
separator = Gtk.SeparatorMenuItem()
separator.set_property("name", "separator")
menu.append(separator)
# actual system menu with submenus for each category
if c_audio_video:
append_submenu(c_audio_video, menu, 'AudioVideo')
if c_development:
append_submenu(c_development, menu, 'Development')
if c_game:
append_submenu(c_game, menu, 'Game')
if c_graphics:
append_submenu(c_graphics, menu, 'Graphics')
if c_network:
append_submenu(c_network, menu, 'Network')
if c_office:
append_submenu(c_office, menu, 'Office')
if c_science:
append_submenu(c_science, menu, 'Science')
if c_settings:
append_submenu(c_settings, menu, 'Settings')
if c_system:
append_submenu(c_system, menu, 'System')
if c_utility:
append_submenu(c_utility, menu, 'Utility')
if c_other:
append_submenu(c_other, menu, 'Other')
# user-defined menu from default or custom file (see args)
if args.append or args.af or args.no_menu:
if not args.no_menu: # nothing above to separate
separator = Gtk.SeparatorMenuItem()
separator.set_property("name", "separator")
menu.append(separator)
appendix = load_json(build_from_file)
for entry in appendix:
name = entry["name"]
exec = entry["exec"]
icon = entry["icon"]
hbox = Gtk.HBox()
label = Gtk.Label()
label.set_text(name)
image = None
if icon.startswith('/'):
try:
pixbuf = GdkPixbuf.Pixbuf.new_from_file_at_size(
icon, args.s, args.s)
image = Gtk.Image.new_from_pixbuf(pixbuf)
except:
pass
else:
try:
if icon.endswith('.svg') or icon.endswith('.png'):
icon = entry.icon.split('.')[0]
pixbuf = icon_theme.load_icon(
icon, args.s, Gtk.IconLookupFlags.FORCE_SIZE)
image = Gtk.Image.new_from_pixbuf(pixbuf)
except:
pass
if image:
hbox.pack_start(image, False, False, 10)
if name:
hbox.pack_start(label, False, False, 0)
item = Gtk.MenuItem()
item.set_property("name", "item-appendix")
item.add(hbox)
item.connect('activate', launch, exec, True) # do not cache!
menu.append(item)
menu.connect("hide", win.die)
menu.set_property("reserve_toggle_size", False)
menu.show_all()
return menu
def main():
# exit if already running, thanks to Slava V at https://stackoverflow.com/a/384493/4040598
pid_file = os.path.join(tempfile.gettempdir(), 'sgtk-menu.pid')
fp = open(pid_file, 'w')
try:
fcntl.lockf(fp, fcntl.LOCK_EX | fcntl.LOCK_NB)
except IOError:
subprocess.run("pkill -f sgtk-menu", shell=True)
sys.exit(2)
global build_from_file
parser = argparse.ArgumentParser(
description="GTK menu for sway, i3 and some floating WMs")
placement = parser.add_mutually_exclusive_group()
placement.add_argument("-b",
"--bottom",
action="store_true",
help="display menu at the bottom (sway & i3 only)")
placement.add_argument("-c",
"--center",
action="store_true",
help="center menu on the screen (sway & i3 only)")
favourites = parser.add_mutually_exclusive_group()
favourites.add_argument("-f",
"--favourites",
action="store_true",
help="prepend 5 most used items")
favourites.add_argument('-fn',
type=int,
help="prepend <FN> most used items")
appendix = parser.add_mutually_exclusive_group()
appendix.add_argument(
"-a",
"--append",
action="store_true",
help="append custom menu from {}".format(build_from_file))
appendix.add_argument("-af",
type=str,
help="append custom menu from {}".format(
os.path.join(config_dir, '<AF>')))
parser.add_argument("-n",
"--no-menu",
action="store_true",
help="skip menu, display appendix only")
parser.add_argument("-l",
type=str,
help="force language (e.g. \"de\" for German)")
parser.add_argument("-s",
type=int,
default=20,
help="menu icon size (min: 16, max: 48, default: 20)")
parser.add_argument(
"-w",
type=int,
help="menu width in px (integer, default: screen width / 8)")
parser.add_argument(
"-d",
type=int,
default=100,
help="menu delay in milliseconds (default: 100; sway & i3 only)")
parser.add_argument(
"-o",
type=float,
default=0.3,
help="overlay opacity (min: 0.0, max: 1.0, default: 0.3; "
"sway & i3 only)")
parser.add_argument("-t",
type=int,
default=30,
help="sway submenu lines limit (default: 30)")
parser.add_argument(
"-y",
type=int,
default=0,
help="y offset from edge to display menu at (sway & i3 only)")
parser.add_argument(
"-css",
type=str,
default="style.css",
help="use alternative {} style sheet instead of style.css".format(
os.path.join(config_dir, '<CSS>')))
global args
args = parser.parse_args()
css_file = os.path.join(config_dirs()[0], args.css) if os.path.exists(
os.path.join(config_dirs()[0], 'style.css')) else None
if args.s < 16:
args.s = 16
elif args.s > 48:
args.s = 48
# We do not need any delay in other WMs
if other_wm:
args.d = 0
# Create default config files if not found
create_default_configs(config_dir)
# Replace appendix file name with custom - if any
if args.af:
build_from_file = os.path.join(config_dirs()[0], args.af)
if css_file:
screen = Gdk.Screen.get_default()
provider = Gtk.CssProvider()
try:
provider.load_from_path(css_file)
Gtk.StyleContext.add_provider_for_screen(
screen, provider, Gtk.STYLE_PROVIDER_PRIORITY_APPLICATION)
except Exception as e:
print(e)
# cache stores number of clicks on each item
global cache
cache = load_json(cache_file)
if not cache:
save_json(cache, cache_file)
global sorted_cache
sorted_cache = sorted(cache.items(), reverse=True, key=lambda x: x[1])
global locale
locale = get_locale_string(args.l)
category_names_dictionary = localized_category_names(locale)
# replace additional category names with main ones
for name in category_names:
main_category_name = additional_to_main(name)
try:
localized_names_dictionary[
main_category_name] = category_names_dictionary[
main_category_name]
except:
pass
screen = Gdk.Screen.get_default()
provider = Gtk.CssProvider()
style_context = Gtk.StyleContext()
style_context.add_provider_for_screen(
screen, provider, Gtk.STYLE_PROVIDER_PRIORITY_APPLICATION)
# find all .desktop entries, create DesktopEntry class instances;
# DesktopEntry adds itself to the proper List in the class constructor
list_entries()
# Overlay window
global win
win = MainWindow()
if other_wm:
# We need this to obtain the screen geometry when i3ipc module unavailable
win.resize(1, 1)
win.show_all()
global geometry
# If we're not on sway neither i3, this won't return values until the window actually shows up.
# Let's try as many times as needed. The retries int protects from an infinite loop.
retries = 0
while geometry[0] == 0 and geometry[1] == 0 and geometry[
2] == 0 and geometry[3] == 0:
geometry = display_geometry()
retries += 1
if retries > 500:
print("\nFailed to get the current screen geometry, exiting...\n")
sys.exit(2)
x, y, w, h = geometry
if not other_wm:
win.resize(w, h)
else:
win.resize(1, 1)
win.set_gravity(Gdk.Gravity.CENTER)
if pynput:
x, y = mouse_pointer.position
win.move(x, y)
else:
win.move(0, 0)
print("\nYou need the python-pynput package!\n")
win.set_skip_taskbar_hint(True)
win.menu = build_menu()
win.menu.set_property("name", "menu")
global menu_items_list
menu_items_list = win.menu.get_children()
win.menu.propagate_key_event = False
win.menu.connect("key-release-event", win.search_items)
# Let's reserve some width for long entries found with the search box
if args.w:
win.menu.set_property("width_request", args.w)
else:
win.menu.set_property("width_request",
int(win.screen_dimensions[0] / 8))
win.show_all()
GLib.timeout_add(args.d, open_menu)
Gtk.main()
import logging
logging.basicConfig(format='%(process)d-%(levelname)s-%(message)s')
# input args
parser = argparse.ArgumentParser()
parser.add_argument("--output_path", help="path to save metadata to", action="store", required=True)
parser.add_argument("--video_metadata_path", help="path to video metadata", action="store", required=True)
args = parser.parse_args()
output_data_path = args.output_path
video_metadata_path = args.video_metadata_path
video_metadata = tools.load_json(video_metadata_path)
authors_distribution = Counter(chain.from_iterable(map(lambda x: x["author_names"],
chain.from_iterable(video_metadata.values())))).most_common()
duration_distribution = Counter(map(lambda x: x["duration"],
chain.from_iterable(video_metadata.values()))).most_common()
youtube_id_distribution = Counter(map(lambda x: x["youtube_id"],
chain.from_iterable(video_metadata.values()))).most_common()
keywords_distribution = Counter(chain.from_iterable(map(lambda x: x["keywords"].split(','),
chain.from_iterable(video_metadata.values())))).most_common()
tools.save_json(os.path.join(output_data_path, 'authors_distribution.json'), authors_distribution)
def main():
parser = argparse.ArgumentParser(description='JSON file')
parser.add_argument("--path",
dest="json_path",
type=str,
help='path to json file. defaults to params.json',
default="params.json")
args = parser.parse_args()
print(args.json_path)
params = load_json(args.json_path)
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
if params["dataset"] == "CUB":
preprocess = transforms.Compose(
[transforms.Resize(256),
transforms.ToTensor(), normalize])
from configs.config_CUB import MAIN_DATAPATH, TEST_DATAPATH
att_split = sio.loadmat(params["CUB_paths"]["att_split"])
root = params["CUB_paths"]["root"]
metaData = sio.loadmat(params["CUB_paths"]["metaData"])
print("CUB Dataset chosen.")
dataloader_placeholder = CUBDataset
elif params["dataset"] == "AWA2":
preprocess = transforms.Compose([
transforms.Resize(256),
transforms.ToTensor(),
])
from configs.config_AWA2 import MAIN_DATAPATH, TEST_DATAPATH
att_split = sio.loadmat(params["AWA2_paths"]["att_split"])
root = params["AWA2_paths"]["root"]
metaData = sio.loadmat(params["AWA2_paths"]["metaData"])
dataloader_placeholder = AWADataset
print("AWA2 Dataset chosen.")
elif params["dataset"] == "SUN":
preprocess = transforms.Compose(
[transforms.Resize(256),
transforms.ToTensor(), normalize])
from configs.config_SUN import MAIN_DATAPATH, TEST_DATAPATH
att_split = sio.loadmat(params["SUN_paths"]["att_split"])
root = params["SUN_paths"]["root"]
metaData = sio.loadmat(params["SUN_paths"]["metaData"])
print("SUN Dataset chosen.")
dataloader_placeholder = SunDataset
else:
print("Invalid dataset chosen. ")
sys.exit()
all_class_vector = load_data(MAIN_DATAPATH + 'all_class_vec.mat',
"all_class_vec")
train_val_labels = load_data(TEST_DATAPATH + 'trainval_labels.mat',
'trainval_labels')
unseen_labels = load_data(TEST_DATAPATH + 'test_unseen_labels.mat',
'test_unseen_labels')
seenClassIndices = np.unique(train_val_labels)
unseenClassIndices = np.unique(unseen_labels)
unseenVectors = torch.from_numpy(
all_class_vector[unseenClassIndices, :]).float().cuda()
allVectors = torch.from_numpy(all_class_vector).float().cuda()
trainval_indexes = att_split["trainval_loc"]
test_unseen_indexes = att_split["test_unseen_loc"]
test_seen_indexes = att_split["test_seen_loc"]
files = metaData["image_files"]
labels = metaData["labels"]
dataloader_zsl = DataLoader(dataloader_placeholder(test_unseen_indexes,
files,
labels,
root,
zsl=True,
transform=preprocess),
batch_size=1,
shuffle=params["shuffle_dataset"],
num_workers=params["num_workers"],
pin_memory=params["pin_memory"])
dataloader_unseen = DataLoader(dataloader_placeholder(
test_unseen_indexes, files, labels, root, transform=preprocess),
batch_size=1,
shuffle=params["shuffle_dataset"],
num_workers=params["num_workers"],
pin_memory=params["pin_memory"])
dataloader_seen = DataLoader(dataloader_placeholder(test_seen_indexes,
files,
labels,
root,
transform=preprocess),
batch_size=1,
shuffle=params["shuffle_dataset"],
num_workers=params["num_workers"],
pin_memory=params["pin_memory"])
criterion = torch.nn.CrossEntropyLoss(reduction='sum')
if params["zsl_test"]:
zsl_launch(dataloader_zsl, unseenVectors, criterion, params)
print("-------" * 20)
if params["gzsl_test"]:
gzsl_launch(dataloader_seen, dataloader_unseen, allVectors, criterion,
params)
def main():
# exit if already running, thanks to Slava V at https://stackoverflow.com/a/384493/4040598
pid_file = os.path.join(tempfile.gettempdir(), 'sgtk-menu.pid')
fp = open(pid_file, 'w')
try:
fcntl.lockf(fp, fcntl.LOCK_EX | fcntl.LOCK_NB)
except IOError:
sys.exit(0)
global appendix_file
parser = argparse.ArgumentParser(description="GTK menu for sway and i3")
parser.add_argument("-b",
"--bottom",
action="store_true",
help="display menu at the bottom")
favourites = parser.add_mutually_exclusive_group()
favourites.add_argument("-f",
"--favourites",
action="store_true",
help="prepend 5 most used items")
favourites.add_argument('-fn',
type=int,
help="prepend <FN> most used items")
appenxid = parser.add_mutually_exclusive_group()
appenxid.add_argument(
"-a",
"--append",
action="store_true",
help="append custom menu from {}".format(appendix_file))
appenxid.add_argument("-af",
type=str,
help="append custom menu from {}".format(
os.path.join(config_dir, '<AF>')))
parser.add_argument("-l",
type=str,
help="force language (e.g. \"de\" for German)")
parser.add_argument("-s",
type=int,
default=20,
help="menu icon size (min: 16, max: 48, default: 20)")
parser.add_argument(
"-w",
type=int,
help="menu width in px (integer, default: screen width / 8)")
parser.add_argument("-d",
type=int,
default=100,
help="menu delay in milliseconds (default: 100)")
parser.add_argument(
"-o",
type=float,
default=0.3,
help="overlay opacity (min: 0.0, max: 1.0, default: 0.3)")
parser.add_argument("-t",
type=int,
default=30,
help="sway submenu lines limit (default: 30)")
global args
args = parser.parse_args()
if args.s < 16:
args.s = 16
elif args.s > 48:
args.s = 48
# Create default appendix file if not found
if not os.path.isfile(appendix_file):
save_default_appendix(appendix_file)
# Replace appendix file name with custom - if any
if args.af:
appendix_file = os.path.join(config_dirs()[0], args.af)
# cache stores number of clicks on each item
global cache
cache = load_json(cache_file)
if not cache:
save_json(cache, cache_file)
global sorted_cache
sorted_cache = sorted(cache.items(), reverse=True, key=lambda x: x[1])
global locale
locale = get_locale_string(args.l)
category_names_dictionary = localized_category_names(locale)
# replace additional category names with main ones
for name in category_names:
main_category_name = additional_to_main(name)
try:
localized_names_dictionary[
main_category_name] = category_names_dictionary[
main_category_name]
except:
pass
screen = Gdk.Screen.get_default()
provider = Gtk.CssProvider()
style_context = Gtk.StyleContext()
style_context.add_provider_for_screen(
screen, provider, Gtk.STYLE_PROVIDER_PRIORITY_APPLICATION)
# find all .desktop entries, create DesktopEntry class instances;
# DesktopEntry adds itself to the proper List in the class constructor
list_entries()
# Overlay window
global win
win = MainWindow()
w, h = display_dimensions()
win.resize(w, h)
win.menu = build_menu()
global menu_items_list
menu_items_list = win.menu.get_children()
win.menu.propagate_key_event = False
win.menu.connect("key-release-event", win.search_items)
# Let's reserve some width for long entries found with the search box
if args.w:
win.menu.set_property("width_request", args.w)
else:
win.menu.set_property("width_request",
int(win.screen_dimensions[0] / 8))
win.show_all()
GLib.timeout_add(args.d, open_menu)
Gtk.main()
def run(self):
like_count = 0
# hashtags_dict_analysis = {'searched_hashtags': {}} #TODO
# Forbidden hashtags
forbidden = tools.load_json(
os.path.join(os.getcwd(), 'forbidden_hashtags.json'))
forbidden = [str(i) for i in forbidden['hashtags']]
driver = webdriver.Chrome()
driver.maximize_window()
driver.implicitly_wait(5)
url = 'https://www.instagram.com/?hl=en'
driver.get(url)
# Access to log in page
driver.find_element_by_xpath(
'//*[@id="react-root"]/section/main/article/div[2]/div[2]/p/a'
).click()
driver.implicitly_wait(5)
# Log in task
log = tools.load_json(os.path.join(os.getcwd(), 'info.json'))
tools.login(driver, log)
driver.implicitly_wait(5)
# Search bar
while True: #loop pour pick le hashtag randomly
like_limit = random.randint(25, self.max_like_per_round)
hashtag = random.choice(log['search'])
# hashtags_dict_analysis['searched_hashtags'][hashtag] = {}
search = driver.find_element_by_css_selector(
'input[placeholder="Search"]') # find Search bar
search.send_keys(hashtag)
driver.implicitly_wait(5)
# Click on first link
driver.find_element_by_xpath(
'//*[@id="react-root"]/section/nav/div[2]/div/div/div[2]/div[2]/div[2]/div/a[1]'
).click()
driver.implicitly_wait(5)
# Click on 'Load More'
try:
driver.execute_script(
"window.scrollTo(0, document.body.scrollHeight);")
driver.find_element_by_link_text('Load more').click()
except NoSuchElementException:
pass
# Click on first photo
first_photo_xpath = '//*[@id="react-root"]/section/main/article/div[2]/div[1]/div[1]/div[1]/a'
first_photo = driver.find_element_by_xpath(first_photo_xpath)
ActionChains(driver).move_to_element(first_photo).perform(
) # remonter vers la 1er image de 'Most recent'
count_row_scroll = 0
# hashtag_count = 1
try:
row = 1
while True: # tant qu'on peut scroll
count_row_scroll += 1
for col in [1, 2, 3]:
col = str(col)
image_i_xpath = '//*[@id="react-root"]/section/main/article/div[2]/div[1]/div[' + str(
row) + ']/div[' + col + ']/a'
image_i = driver.find_element_by_xpath(image_i_xpath)
ActionChains(driver).move_to_element(image_i).perform()
image_i.click()
username_xpath = '/html/body/div[3]/div/div/div[2]/div/article/header/div[2]/div[1]/div/a'
username = driver.find_element_by_xpath(
username_xpath).get_attribute('title')
if 'shop' in username: # TODO A RETRAVAILLER
pass
else:
caption_xpath = '/html/body/div[3]/div/div/div[2]/div/article/div[2]/div[1]/ul/li'
caption = driver.find_element_by_xpath(
caption_xpath).text
# Get hashtags with regex
# hashtags_ = re.findall(r"#(\w+)", caption)
# hashtags_ = list(set(hashtags_))
# any_in = lambda a, b: bool(set(a).intersection(b)) # intersection between 2 lists
# if any_in(forbidden, hashtags_):
# pass
# else:
# hashtags_dict_analysis['searched_hashtags'][hashtag] = hashtags_
# hashtag_count += 1
# Follow, like
try: # a mettre dans le else
if driver.find_element_by_xpath(
"//button[contains(.,'Following')]"):
print(username + 'is already followed')
except NoSuchElementException:
driver.find_element_by_xpath(
"//button[contains(.,'Follow')]").click()
time.sleep(random.uniform(0.2, 0.8))
try:
driver.find_element_by_xpath(
"//span[contains(.,'Like')]").click()
like_count += 1
# hashtag_count += 1
except NoSuchElementException: # if already liked [contains(., 'Unlike)]
pass
finally:
time.sleep(random.uniform(0.2, 0.8))
driver.find_element_by_xpath(
"//button[contains(.,'Close')]").click()
driver.implicitly_wait(5)
# Scroll down
if count_row_scroll == 4: # 1 new load = 4 lignes de charger
driver.execute_script(
"window.scrollTo(0, document.body.scrollHeight);")
time.sleep(random.uniform(0.3, 0.8))
count_row_scroll = 0
row += 1
if like_count == like_limit: # tous les X likes, stop
stop_ = random.uniform(20, 45)
print(time.localtime(), stop_)
time.sleep(60 * stop_)
like_limit = random.randint(25,
self.max_like_per_round)
print('Onto the next : %s' % (time.strftime(
'%d/%m/20%y_%H:%M:%S', time.localtime())))
except NoSuchElementException: # if no more image
pass # look for another hashtag in search bar
if time.localtime()[7] == self.start[7]: #si meme jour
if time.localtime()[3] == self.start[3] + 10:
self.restart()
else:
self.restart()
import argparse
import logging
logging.basicConfig(format='%(process)d-%(levelname)s-%(message)s')
# input args
parser = argparse.ArgumentParser()
parser.add_argument("--output_path", help="path to save metadata to", action="store", required=True)
parser.add_argument("--topic_tree_backup", help="path to backup of Khan academy topic tree", action="store", required=True)
args = parser.parse_args()
output_data_path = args.output_path
topic_tree_backup = args.topic_tree_backup
topic_tree = tools.load_json(topic_tree_backup)
metadata_mask = ['author_names', 'creation_date', 'date_added', 'description', 'description_html', 'duration',
'image_url', 'ka_url', 'keywords', 'readable_id', 'slug', 'thumbnail_urls', 'title',
'translated_description', 'translated_description_html', 'translated_title', 'translated_youtube_id',
'translated_youtube_lang', 'youtube_id']
topic_tree_leafs = tools.get_leafs(topic_tree)
logging.info('Gathering video metadata')
video_metadata = defaultdict(list)
for leaf in topic_tree_leafs:
video_metadata[leaf['youtube_id']].append({key: leaf[key] for key in metadata_mask})
video_metadata = dict(video_metadata)
import docopt
import requests
import os
import logging
from tools import load_json
from pkg_resources import resource_filename
from toolz import partition_all, compose
from functools import partial
from tools import load_json_lines, write_json_lines, post_json, log_stream, key_to_pos
# google location api
g_map = "https://maps.googleapis.com/maps/api/{method}/json?"
google_key = os.environ['GOOGLE_MAPS_KEY']
continent_map = load_json(resource_filename(__name__,
'country_continent.json'))
def _extract_name_from_affiliations(affiliations):
for a in affiliations:
yield a['info']
def get_location(info):
params = {'key': google_key, 'query': info['Name']}
url = g_map.format(method='place/textsearch')
resp = requests.get(url=url, params=params)
resp = resp.json()
if len(resp["results"]):
address = resp["results"][0]["formatted_address"]
import numpy as np
import matplotlib.pyplot as plt
from tools import load_json
plt.switch_backend('agg') # 防止ssh上绘图问题
data1 = load_json('./logs/ResNet18_training_monitor.json')
data2 = load_json('./logs/ResNet18_label_smoothing_training_monitor.json')
N = np.arange(0, len(data1['loss']))
plt.style.use("ggplot")
plt.figure()
plt.plot(N, data1['loss'], label=f"ResNet18")
plt.plot(N, data2['loss'], label=f"ResNet18_label_smooth")
plt.legend()
plt.xlabel("Epoch #")
plt.ylabel('loss')
plt.title(f"Training loss [Epoch {len(data1['loss'])}]")
plt.savefig('./png/training_loss.png')
plt.close()
N = np.arange(0, len(data1['loss']))
plt.style.use("ggplot")
plt.figure()
plt.plot(N, data1['valid_acc'], label=f"ResNet18")
plt.plot(N, data2['valid_acc'], label=f"ResNet18_label_smooth")
plt.legend()
plt.xlabel("Epoch #")
plt.ylabel('accuracy')
plt.title(f"Valid accuracy [Epoch {len(data1['loss'])}]")
plt.savefig('./png/valid_accuracy.png')
plt.close()
def create_aff_body(ids):
return {
"path": "/aff",
"aff": {
"type": "Affiliation",
"id": ids,
"select": ["Name"],
}
}
g_map = "https://maps.googleapis.com/maps/api/{method}/json?"
google_key = os.environ['GOOGLE_MAPS_KEY']
continent_map = load_json('country_continent.json')
def get_location(name):
params = {'key': google_key, 'query': name}
url = g_map.format(method='place/textsearch')
resp = requests.get(url=url, params=params)
resp = resp.json()
if len(resp["results"]):
address = resp["results"][0]["formatted_address"]
params = {'key': google_key, 'address': address}
url = g_map.format(method='geocode')
resp = requests.get(url=url, params=params)
resp = resp.json()
