def getAndUpdateFullRevisionList(self, node_test_suite): full_revision_list = [] config = self.config log = self.log try: for vcs_repository in node_test_suite.vcs_repository_list: repository_path = vcs_repository['repository_path'] repository_id = vcs_repository['repository_id'] branch = vcs_repository.get('branch') # Make sure we have local repository updater = Updater( repository_path, git_binary=config['git_binary'], branch=branch, log=log, process_manager=self.process_manager, working_directory=node_test_suite.working_directory, url=vcs_repository["url"]) updater.checkout() revision = "-".join(updater.getRevision()) full_revision_list.append('%s=%s' % (repository_id, revision)) node_test_suite.revision = ','.join(full_revision_list) except SubprocessError, e: log("Error while getting repository, ignoring this test suite : %r" % (e, ), exc_info=sys.exc_info()) full_revision_list = None
def __init__(self): self.updater_client = Updater("Urls.config") self.email_content = self.set_up_email_content() # set up the SMTP server self.email, self.password = self.get_creditionals( "Creditionals.config") self.server = smtplib.SMTP("smtp-mail.outlook.com", 587) #self.server = smtplib.SMTP("smtp.gmail.com", 587) self.server.starttls() self.server.login(self.email, self.password)
def get_update(): instance = int(request.args.get("instance")) if instance == 1: return to_JSON( UpdateAvailable(Updater().check_version(), Updater().last_version)) else: result = APIController().slave_request(instance, "get_last_version", 10) if result is None: return to_JSON(UpdateAvailable(False, "")) return to_JSON(UpdateAvailable(result[0], result[1]))
def updatePlugin(): global sessionId stater = Stater(sessionId) updater = Updater(sessionId) while True: time.sleep(updater.getPool()) info = stater.getCurrentService() if "name" in info: #Update only when watching TV updater.updateRequests() if sessionId == None: break
def __init__(self): self.client = None self.__loadMonitors() try: app_key, app_secret = self.__getAppKeyAndSecret() except Exception as e: Dialog(visual).error("Could not contact key server", "Could not contact key server, unable to verify app. \n%s" % e) return self.flow = dropbox.client.DropboxOAuth2FlowNoRedirect(app_key, app_secret) self.loadSettings() update = Updater() update.update(self, visual)
def checkRevision(self, test_result, node_test_suite): config = self.config log = self.log if log is None: log = self.log if node_test_suite.revision != test_result.revision: log('Disagreement on tested revision, checking out: %r' % ((node_test_suite.revision, test_result.revision), )) for i, repository_revision in enumerate( test_result.revision.split(',')): vcs_repository = node_test_suite.vcs_repository_list[i] repository_path = vcs_repository['repository_path'] revision = repository_revision.rsplit('-', 1)[1] # other testnodes on other boxes are already ready to test another # revision log(' %s at %s' % (repository_path, node_test_suite.revision)) updater = Updater(repository_path, git_binary=config['git_binary'], revision=revision, log=log, process_manager=self.process_manager) updater.checkout() updater.git_update_server_info() updater.git_create_repository_link() node_test_suite.revision = test_result.revision
def update(): instance = int(request.args.get("instance")) if instance == 1: Updater().update() else: APIController().slave_command(instance, "updater", "update") return "OK"
def init(): from Controllers.TradfriManager import TradfriManager from MediaPlayer.MediaManager import MediaManager from MediaPlayer.Player.VLCPlayer import VLCPlayer from Updater import Updater from Shared.State import StateManager from Shared.Stats import Stats APIController.slaves.register_callback( lambda old, new: UIWebsocketController.broadcast( "slaves", new.data)) TradfriManager().tradfri_state.register_callback( lambda old, new: UIWebsocketController.broadcast("tradfri", new)) StateManager().state_data.register_callback( lambda old, new: UIWebsocketController.broadcast("1.state", new)) VLCPlayer().player_state.register_callback( lambda old, new: UIWebsocketController.broadcast("1.player", new)) MediaManager().media_data.register_callback( lambda old, new: UIWebsocketController.broadcast("1.media", new)) MediaManager().torrent_data.register_callback( lambda old, new: UIWebsocketController.broadcast("1.torrent", new)) Stats().cache.register_callback( lambda old, new: UIWebsocketController.broadcast("1.stats", new)) Updater().update_state.register_callback( lambda old, new: UIWebsocketController.broadcast("1.update", new))
def init_train_from_config(self, config, train_data, dev_data=None, eval_data=None): """ :type config: Config.Config :type train_data: Dataset.Dataset :type dev_data: Dataset.Dataset | None :type eval_data: Dataset.Dataset | None """ self.train_data = train_data self.dev_data = dev_data self.eval_data = eval_data self.start_epoch, self.start_batch = self.get_train_start_epoch_batch(config) self.batch_size = config.int('batch_size', 1) self.shuffle_batches = config.bool('shuffle_batches', True) self.update_batch_size = config.int('update_batch_size', 0) self.model_filename = config.value('model', None) self.save_model_epoch_interval = config.int('save_interval', 1) self.save_epoch1_initial_model = config.bool('save_epoch1_initial_model', False) self.learning_rate_control = loadLearningRateControlFromConfig(config) self.learning_rate = self.learning_rate_control.defaultLearningRate self.initial_learning_rate = self.learning_rate self.pretrain_learning_rate = config.float('pretrain_learning_rate', self.learning_rate) self.final_epoch = self.config_get_final_epoch(config) # Inclusive. self.max_seqs = config.int('max_seqs', -1) self.updater = Updater.initFromConfig(config) self.ctc_prior_file = config.value('ctc_prior_file', None) self.exclude = config.int_list('exclude', []) self.init_train_epoch_posthook = config.value('init_train_epoch_posthook', None) self.share_batches = config.bool('share_batches', False) self.batch_variance = config.float('batch_variance', 0.0) self.max_seq_length = config.float('max_seq_length', 0) self.inc_seq_length = config.float('inc_seq_length', 0) if self.max_seq_length == 0: self.max_seq_length = sys.maxint # And also initialize the network. That depends on some vars here such as pretrain. self.init_network_from_config(config)
def init_train_from_config(self, config, train_data, dev_data=None, eval_data=None): """ :type config: Config.Config :type train_data: Dataset.Dataset :type dev_data: Dataset.Dataset | None :type eval_data: Dataset.Dataset | None """ self.train_data = train_data self.dev_data = dev_data self.eval_data = eval_data self.start_epoch, self.start_batch = self.get_train_start_epoch_batch( config) self.batch_size = config.int('batch_size', 1) self.shuffle_batches = config.bool('shuffle_batches', False) self.update_batch_size = config.float('update_batch_size', 0) self.batch_size_eval = config.int('batch_size_eval', self.update_batch_size) self.model_filename = config.value('model', None) self.save_model_epoch_interval = config.int('save_interval', 1) self.save_epoch1_initial_model = config.bool( 'save_epoch1_initial_model', False) self.learning_rate_control = load_learning_rate_control_from_config( config) self.learning_rate = self.learning_rate_control.default_learning_rate self.initial_learning_rate = self.learning_rate self.pretrain_learning_rate = config.float('pretrain_learning_rate', self.learning_rate) self.final_epoch = self.config_get_final_epoch(config) # Inclusive. self.max_seqs = config.int('max_seqs', -1) self.max_seqs_eval = config.int('max_seqs_eval', self.max_seqs) self.updater = Updater.initFromConfig(config) self.ctc_prior_file = config.value('ctc_prior_file', None) self.exclude = config.int_list('exclude', []) self.init_train_epoch_posthook = config.value( 'init_train_epoch_posthook', None) self.share_batches = config.bool('share_batches', False) self.seq_drop = config.float('seq_drop', 0.0) self.seq_drop_freq = config.float('seq_drop_freq', 10) self.max_seq_length = config.float('max_seq_length', 0) self.inc_seq_length = config.float('inc_seq_length', 0) self.max_seq_length_eval = config.int('max_seq_length_eval', 2e31) self.output_precision = config.int('output_precision', 12) self.reduction_rate = config.float('reduction_rate', 1.0) self.batch_pruning = config.float('batch_pruning', 0.0) if self.max_seq_length == 0: self.max_seq_length = sys.maxsize if config.is_typed("seq_train_parallel"): self.seq_train_parallel = SeqTrainParallelControl( engine=self, config=config, **config.typed_value("seq_train_parallel")) else: self.seq_train_parallel = None # And also initialize the network. That depends on some vars here such as pretrain. self.init_network_from_config(config)
def __init__(self, editor): GObject.__init__(self) self.__init_attributes(editor) from Updater import Updater Updater(editor, self) from TreeView import TreeView TreeView(editor, self) from Window import Window Window(editor, self)
def getAndUpdateFullRevisionList(self, node_test_suite): full_revision_list = [] config = self.config log = self.log for vcs_repository in node_test_suite.vcs_repository_list: repository_path = vcs_repository['repository_path'] repository_id = vcs_repository['repository_id'] branch = vcs_repository.get('branch') # Make sure we have local repository updater = Updater(repository_path, git_binary=config['git_binary'], branch=branch, log=log, process_manager=self.process_manager, working_directory=node_test_suite.working_directory, url=vcs_repository["url"]) updater.checkout() revision = "-".join(updater.getRevision()) full_revision_list.append('%s=%s' % (repository_id, revision)) node_test_suite.revision = ','.join(full_revision_list) return full_revision_list
def checkRevision(self, test_result, node_test_suite): config = self.config log = self.log if node_test_suite.revision != test_result.revision: log('Disagreement on tested revision, checking out: %r' % ( (node_test_suite.revision,test_result.revision),)) for i, repository_revision in enumerate(test_result.revision.split(',')): vcs_repository = node_test_suite.vcs_repository_list[i] repository_path = vcs_repository['repository_path'] revision = repository_revision.rsplit('-', 1)[1] # other testnodes on other boxes are already ready to test another # revision log(' %s at %s' % (repository_path, node_test_suite.revision)) updater = Updater(repository_path, git_binary=config['git_binary'], revision=revision, log=log, process_manager=self.process_manager) updater.checkout() node_test_suite.revision = test_result.revision
def init(self): self = objc.super(BlinkAppDelegate, self).init() if self: self.applicationName = str(NSBundle.mainBundle().infoDictionary().objectForKey_("CFBundleExecutable")) self.applicationNamePrint = str(NSBundle.mainBundle().infoDictionary().objectForKey_("CFBundleName")) build = str(NSBundle.mainBundle().infoDictionary().objectForKey_("CFBundleVersion")) date = str(NSBundle.mainBundle().infoDictionary().objectForKey_("BlinkVersionDate")) branding_file = NSBundle.mainBundle().infoDictionary().objectForKey_("BrandingFile") try: branding = __import__(branding_file) except ImportError: try: import branding except ImportError: branding = Null branding.init(self) BlinkLogger().log_info("Starting %s %s" % (self.applicationNamePrint, build)) self.registerURLHandler() NSWorkspace.sharedWorkspace().notificationCenter().addObserver_selector_name_object_(self, "computerDidWake:", NSWorkspaceDidWakeNotification, None) NSWorkspace.sharedWorkspace().notificationCenter().addObserver_selector_name_object_(self, "computerWillSleep:", NSWorkspaceWillSleepNotification, None) NSDistributedNotificationCenter.defaultCenter().addObserver_selector_name_object_suspensionBehavior_(self, "callFromAddressBook:", "CallTelephoneNumberWithBlinkFromAddressBookNotification", "AddressBook", NSNotificationSuspensionBehaviorDeliverImmediately) NSDistributedNotificationCenter.defaultCenter().addObserver_selector_name_object_suspensionBehavior_(self, "callFromAddressBook:", "CallSipAddressWithBlinkFromAddressBookNotification", "AddressBook", NSNotificationSuspensionBehaviorDeliverImmediately) NotificationCenter().add_observer(self, name="CFGSettingsObjectDidChange") NotificationCenter().add_observer(self, name="SIPApplicationDidStart") NotificationCenter().add_observer(self, name="SIPApplicationWillEnd") NotificationCenter().add_observer(self, name="SIPApplicationDidEnd") NotificationCenter().add_observer(self, name="NetworkConditionsDidChange") NotificationCenter().add_observer(self, name="SIPEngineTransportDidDisconnect") NotificationCenter().add_observer(self, name="SIPEngineTransportDidConnect") NotificationCenter().add_observer(self, name="DNSNameserversDidChange") NotificationCenter().add_observer(self, name="SystemDidWakeUpFromSleep") # remove obsolete settings userdef = NSUserDefaults.standardUserDefaults() userdef.removeObjectForKey_('SIPTrace') userdef.removeObjectForKey_('MSRPTrace') userdef.removeObjectForKey_('XCAPTrace') userdef.removeObjectForKey_('EnablePJSIPTrace') userdef.removeObjectForKey_('EnableNotificationsTrace') try: from Updater import Updater except ImportError: pass else: self.updater = Updater() self.purge_temporary_files() return self
def init_singletons(): Stats() VLCPlayer() NextEpisodeManager() WiFiController() MediaManager() Updater() ThreadManager() PresenceManager() RuleManager()
def run(self): """Moves the Updater module to a separate thread and runs it""" self.update_label('Starting updater...') self.running_gui_shift() self.thread = QtCore.QThread() self.updater = Updater(self.download_url, self.download_name, self.new_version, self.program_files_location) self.cancel_signal.connect(self.updater.stop) self.updater.moveToThread(self.thread) self.thread.started.connect(self.updater.run_update) self.updater.update_label.connect(self.update_label) self.updater.setup_progress_bar.connect(self.setup_progress_bar) self.updater.update_progress_bar.connect(self.update_progress_bar) self.updater.error_signal.connect(self.update_error) self.updater.finished.connect(self.thread.quit) self.updater.finished.connect(self.updater.deleteLater) self.updater.finished.connect(self.finished_gui_shift) self.thread.finished.connect(self.thread.deleteLater) self.thread.start()
def getAndUpdateFullRevisionList(self, node_test_suite): full_revision_list = [] config = self.config log = self.log try: for vcs_repository in node_test_suite.vcs_repository_list: repository_path = vcs_repository['repository_path'] repository_id = vcs_repository['repository_id'] branch = vcs_repository.get('branch') # Make sure we have local repository updater = Updater(repository_path, git_binary=config['git_binary'], branch=branch, log=log, process_manager=self.process_manager, working_directory=node_test_suite.working_directory, url=vcs_repository["url"]) updater.checkout() revision = "-".join(updater.getRevision()) full_revision_list.append('%s=%s' % (repository_id, revision)) node_test_suite.revision = ','.join(full_revision_list) except SubprocessError, e: log("Error while getting repository, ignoring this test suite : %r" % (e,), exc_info=sys.exc_info()) full_revision_list = None
def getAndUpdateFullRevisionList(self, node_test_suite): full_revision_list = [] config = self.config log = self.log for vcs_repository in node_test_suite.vcs_repository_list: repository_path = vcs_repository['repository_path'] repository_id = vcs_repository['repository_id'] branch = vcs_repository.get('branch') # Make sure we have local repository updater = Updater( repository_path, git_binary=config['git_binary'], branch=branch, log=log, process_manager=self.process_manager, working_directory=node_test_suite.working_directory, url=vcs_repository["url"]) updater.checkout() revision = "-".join(updater.getRevision()) full_revision_list.append('%s=%s' % (repository_id, revision)) node_test_suite.revision = ','.join(full_revision_list) return full_revision_list
def getAndUpdateFullRevisionList(self, node_test_suite): full_revision_list = [] config = self.config log = self.log for vcs_repository in node_test_suite.vcs_repository_list: repository_path = vcs_repository['repository_path'] repository_id = vcs_repository['repository_id'] if not os.path.exists(repository_path): parameter_list = [config['git_binary'], 'clone', vcs_repository['url']] if vcs_repository.get('branch') is not None: parameter_list.extend(['-b',vcs_repository.get('branch')]) parameter_list.append(repository_path) log(subprocess.check_output(parameter_list, stderr=subprocess.STDOUT)) # Make sure we have local repository updater = Updater(repository_path, git_binary=config['git_binary'], log=log, process_manager=self.process_manager) updater.checkout() revision = "-".join(updater.getRevision()) full_revision_list.append('%s=%s' % (repository_id, revision)) node_test_suite.revision = ','.join(full_revision_list) return full_revision_list
def main(argv): def __call__(self): pass stdscr = curses.initscr() curses.noecho() # @UndefinedVariable curses.cbreak() # @UndefinedVariable stdscr.keypad(1) curses.start_color() try: wl = WindowLogic(stdscr) #wl.start() except Exception as e: raise try: dlog = wl.dlog dlog.msg("Logging debug output to " + str(dlog.outputFile)) dlog.msg("Images will be cached in " + wl.cfg.get('file.image.directory')) ci = CommandInterpreter(stdscr, wl) ci.start() updater = Updater(stdscr, wl) updater.start() except Exception as e: dlog.excpt(e) raise ci.join() dlog.msg("Command Interpreter joined.") updater.stop() updater.join() dlog.msg("Updater joined.") #wl.stop() #wl.join() dlog.msg("Thread Fetcher joined.") curses.nocbreak() # @UndefinedVariable stdscr.keypad(0) curses.echo() # @UndefinedVariable curses.endwin() # @UndefinedVariable curses.resetty() # @UndefinedVariable dlog.msg("Terminal restored.")
def updateRevisionList(self, node_test_suite): config = self.config log = self.log revision_list = [] try: for vcs_repository in node_test_suite.vcs_repository_list: repository_path = vcs_repository["repository_path"] repository_id = vcs_repository["repository_id"] branch = vcs_repository.get("branch") # Make sure we have local repository updater = Updater( repository_path, git_binary=config["git_binary"], branch=branch, log=log, process_manager=self.process_manager, working_directory=node_test_suite.working_directory, url=vcs_repository["url"], ) updater.checkout() revision_list.append((repository_id, updater.getRevision())) except SubprocessError, e: log("Error while getting repository, ignoring this test suite", exc_info=1) return False
def init_train_from_config(self, config, train_data, dev_data=None, eval_data=None): """ :type config: Config.Config :type train_data: Dataset.Dataset :type dev_data: Dataset.Dataset | None :type eval_data: Dataset.Dataset | None """ self.train_data = train_data self.dev_data = dev_data self.eval_data = eval_data self.start_epoch, self.start_batch = self.get_train_start_epoch_batch(config) self.batch_size = config.int('batch_size', 1) self.shuffle_batches = config.bool('shuffle_batches', False) self.update_batch_size = config.float('update_batch_size', 0) self.batch_size_eval = config.int('batch_size_eval', self.update_batch_size) self.model_filename = config.value('model', None) self.save_model_epoch_interval = config.int('save_interval', 1) self.save_epoch1_initial_model = config.bool('save_epoch1_initial_model', False) self.learning_rate_control = load_learning_rate_control_from_config(config) self.learning_rate = self.learning_rate_control.default_learning_rate self.initial_learning_rate = self.learning_rate self.pretrain_learning_rate = config.float('pretrain_learning_rate', self.learning_rate) self.final_epoch = self.config_get_final_epoch(config) # Inclusive. self.max_seqs = config.int('max_seqs', -1) self.max_seqs_eval = config.int('max_seqs_eval', self.max_seqs) self.updater = Updater.initFromConfig(config) self.ctc_prior_file = config.value('ctc_prior_file', None) self.exclude = config.int_list('exclude', []) self.init_train_epoch_posthook = config.value('init_train_epoch_posthook', None) self.share_batches = config.bool('share_batches', False) self.seq_drop = config.float('seq_drop', 0.0) self.seq_drop_freq = config.float('seq_drop_freq', 10) self.max_seq_length = config.float('max_seq_length', 0) self.inc_seq_length = config.float('inc_seq_length', 0) self.max_seq_length_eval = config.int('max_seq_length_eval', 2e31) self.output_precision = config.int('output_precision', 12) self.reduction_rate = config.float('reduction_rate', 1.0) self.batch_pruning = config.float('batch_pruning', 0.0) if self.max_seq_length == 0: self.max_seq_length = sys.maxsize if config.is_typed("seq_train_parallel"): self.seq_train_parallel = SeqTrainParallelControl(engine=self, config=config, **config.typed_value("seq_train_parallel")) else: self.seq_train_parallel = None # And also initialize the network. That depends on some vars here such as pretrain. self.init_network_from_config(config)
def init(): if Settings.get_int("log_level") == 0: import logging logging.getLogger('requests').setLevel(logging.WARNING) logging.basicConfig(level=logging.DEBUG) from MediaPlayer.MediaManager import MediaManager from MediaPlayer.Player.VLCPlayer import VLCPlayer from Updater import Updater from Shared.State import StateManager from Shared.Stats import Stats StateManager().state_data.register_callback(lambda old, new: SlaveClientController.broadcast("state", new)) VLCPlayer().player_state.register_callback(lambda old, new: SlaveClientController.broadcast("player", new)) MediaManager().media_data.register_callback(lambda old, new: SlaveClientController.broadcast("media", new)) MediaManager().torrent_data.register_callback(lambda old, new: SlaveClientController.broadcast("torrent", new)) Stats().cache.register_callback(lambda old, new: SlaveClientController.broadcast("stats", new)) Updater().update_state.register_callback(lambda old, new: SlaveClientController.broadcast("update", new))
def init(): bot_id = '1437569240:AAEd2sZ0faC1EwPvQGJPPW4xf7ohP1hTzV8' updater = Updater(bot_id) updater.setPhotoHandler(imageHandler) QualityChecker.init() ShoeDetector.init() FeatureExtractor.init() data_structure = Indexer.build_data_structure(config.DATASET_PATH) Matcher.init(data_structure) print("Bot is running...") updater.start()
def checkRevision(self, test_result, node_test_suite): if node_test_suite.revision == test_result.revision: return log = self.log log("Disagreement on tested revision, checking out: %r != %r", node_test_suite.revision, test_result.revision) updater_kw = dict(git_binary=self.config["git_binary"], log=log, process_manager=self.process_manager) revision_list = [] for i, revision in enumerate(test_result.revision.split(",")): vcs_repository = node_test_suite.vcs_repository_list[i] repository_path = vcs_repository["repository_path"] count, revision = revision.split("=")[1].split("-") revision_list.append((vcs_repository["repository_id"], (int(count), revision))) # other testnodes on other boxes are already ready to test another # revision updater = Updater(repository_path, revision=revision, **updater_kw) updater.checkout() updater.git_update_server_info() updater.git_create_repository_link() node_test_suite.revision_list = revision_list
def on_command(topic, command, args): Logger().write(LogVerbosity.Debug, "Master command " + topic + ": " + command) method = None if topic == "media": from MediaPlayer.MediaManager import MediaManager method = getattr(MediaManager(), command) if topic == "updater": from Updater import Updater method = getattr(Updater(), command) if topic == "system": if command == "restart_device": os.system('sudo reboot') if command == "restart_application": python = sys.executable os.execl(python, python, *sys.argv) if command == "close_application": sys.exit() if method is not None: cb_thread = CustomThread(method, "Master command", args) cb_thread.start()
def __init__(self, parent, *args, **kwargs): tk.Frame.__init__(self, parent, *args, **kwargs) self.parent = parent self.uiMenu = UIMenu(self.parent) self.parent.config(menu=self.uiMenu.menu) # self.toolbar = ttk.Frame(self.parent) # self.toolbarButton1 = ttk.Button(self.toolbar, text='Button') # self.toolbarButton1.pack(side=LEFT, padx=2, pady=2) # self.toolbar.pack(side=TOP, fill=X) self.statusbar = Label(self.parent, text='Ready', bd=1, relief=SUNKEN, anchor=W) self.statusbar.pack(side=BOTTOM, fill=X) self.notebook = ttk.Notebook(self.parent) self.frameSniff = ttk.Frame(self.notebook) self.frameArp = ttk.Frame(self.notebook) self.frameDhcp = ttk.Frame(self.notebook) self.frameSysInfo = ttk.Frame(self.notebook) self.notebook.add(self.frameSniff, text='Sniffer') self.notebook.add(self.frameArp, text='ARP') self.notebook.add(self.frameDhcp, text='DHCP Servers') self.notebook.add(self.frameSysInfo, text='System Info') self.notebook.pack(fill=BOTH) self.sniffLabelFrame = ttk.LabelFrame(self.frameSniff, text="Sniffer") self.sniffLabelFrame.pack(padx=10, pady=10) self.sniffButton = ttk.Button(self.sniffLabelFrame, text="Sniff", command=self.beginSniff) self.sniffButton.pack(side=LEFT) self.stopSniffButton = ttk.Button(self.sniffLabelFrame, text="Stop sniffing", command=self.stopSniff, state="disabled") self.stopSniffButton.pack(side=LEFT) self.sniffTv = ttk.Treeview(self.frameSniff) ysb = ttk.Scrollbar(self, orient='vertical', command=self.sniffTv.yview) xsb = ttk.Scrollbar(self, orient='horizontal', command=self.sniffTv.xview) self.sniffTv.configure(yscroll=ysb.set, xscroll=xsb.set) self.sniffTv['columns'] = ('senderip', 'sendermac', 'received') self.sniffTv.heading('#0', text='Description', anchor='w') self.sniffTv.column('#0', anchor='w') self.sniffTv.heading('senderip', text='Sender IP') self.sniffTv.column('senderip', width=100) self.sniffTv.heading('sendermac', text='Sender MAC') self.sniffTv.column('sendermac', width=100) self.sniffTv.heading('received', text='Received at') self.sniffTv.column('received', width=100) self.sniffTv.pack(fill=BOTH) self.arpLabel = ttk.Label(self.frameArp, text="ARP cache") self.arpLabel.pack() self.arpTv = ttk.Treeview(self.frameArp) self.arpTv['columns'] = ('ip', 'status') self.arpTv.heading('#0', text='MAC address', anchor='w') self.arpTv.column('#0', anchor='w') self.arpTv.heading('ip', text='IP address') self.arpTv.column('ip', width=100) self.arpTv.heading('status', text='Status') self.arpTv.column('status', width=100) self.arpTv.pack(fill=X) self.addDhcpLabelFrame = ttk.LabelFrame(self.frameDhcp, text="Add trusted server") self.addDhcpLabelFrame.pack(padx=10, pady=10) self.addDhcpNameLabel = ttk.Label(self.addDhcpLabelFrame, text="Server name") self.addDhcpNameLabel.pack() self.addDhcpNameEntry = ttk.Entry(self.addDhcpLabelFrame) self.addDhcpNameEntry.pack() self.addDhcpIpLabel = ttk.Label(self.addDhcpLabelFrame, text="Server IP address") self.addDhcpIpLabel.pack() self.addDhcpIpEntry = ttk.Entry(self.addDhcpLabelFrame) self.addDhcpIpEntry.pack() self.addDhcpMacLabel = ttk.Label(self.addDhcpLabelFrame, text="Server Mac address") self.addDhcpMacLabel.pack() self.addDhcpMacEntry = ttk.Entry(self.addDhcpLabelFrame) self.addDhcpMacEntry.pack() self.addDhcpButton = ttk.Button(self.addDhcpLabelFrame, text="Add", command=self.addDhcpButtonPressed) self.addDhcpButton.pack() self.clrDhcpButton = ttk.Button(self.addDhcpLabelFrame, text="Clear", command=self.clrDhcpButtonPressed) self.clrDhcpButton.pack() self.dhcpTv = ttk.Treeview(self.frameDhcp) self.dhcpTv['columns'] = ('ip', 'mac', 'date') self.dhcpTv.heading('#0', text='Server name', anchor='w') self.dhcpTv.column('#0', anchor='w') self.dhcpTv.heading('ip', text='IP address') self.dhcpTv.column('ip', width=100) self.dhcpTv.heading('mac', text='MAC address') self.dhcpTv.column('mac', width=100) self.dhcpTv.heading('date', text='Date added') self.dhcpTv.column('date', width=100) self.dhcpTv.pack(fill=X) strVersion = 'Python ' + platform.python_version() self.versionLabel = ttk.Label(self.frameSysInfo, text=strVersion) self.versionLabel.pack() strPlatform = 'Platform: ' + platform.platform() self.platformLabel = ttk.Label(self.frameSysInfo, text=strPlatform) self.platformLabel.pack() self.dhcpDefender = DhcpDefender(self) self.arpDefender = ArpDefender(self) self.sniffer = Sniffer(1, 'Sniffer-1', 1, self) self.sniffer.start() self.updater = Updater(1, 'Updater-1', 1, self) self.updater.start()
print('Augmenting dataframe with translation columns... ', datetime.now().isoformat()) pm.execute_notebook('EntityTranslCols.ipynb', outdir + '/EntityTranslCols.out.ipynb', parameters=dict(repo=repo_src, version=version, store_data_dir=outdir), kernel_name=kernel) print('Generating entity clusters... ', datetime.now().isoformat()) pm.execute_notebook( cluster_nb, outdir + '/er.out.ipynb', parameters=dict( input_df_path=outdir + '/entity_trans_all_' + version + '.h5', repo_name=repo_src, version=version, kg_tab_dir_path=kg_tab_dir_path, output_path=outdir + '/entity_clusters_' + version + '.jl'), kernel_name=kernel) print('Generating event clusters... ', datetime.now().isoformat()) gen_event_clusters(endpoint_src, outdir + '/event_clusters_' + version + '.jl') print('Insert into GraphDB... ', datetime.now().isoformat()) up = Updater(endpoint_src, endpoint_dst, repo_dst, outdir, graph, True) up.run_all(delete_existing_clusters=delete_existing_clusters, entity_clusters='entity_clusters_' + version + '.jl', event_clusters='event_clusters_' + version + '.jl')
class MainApplication(tk.Frame): def __init__(self, parent, *args, **kwargs): tk.Frame.__init__(self, parent, *args, **kwargs) self.parent = parent self.uiMenu = UIMenu(self.parent) self.parent.config(menu=self.uiMenu.menu) # self.toolbar = ttk.Frame(self.parent) # self.toolbarButton1 = ttk.Button(self.toolbar, text='Button') # self.toolbarButton1.pack(side=LEFT, padx=2, pady=2) # self.toolbar.pack(side=TOP, fill=X) self.statusbar = Label(self.parent, text='Ready', bd=1, relief=SUNKEN, anchor=W) self.statusbar.pack(side=BOTTOM, fill=X) self.notebook = ttk.Notebook(self.parent) self.frameSniff = ttk.Frame(self.notebook) self.frameArp = ttk.Frame(self.notebook) self.frameDhcp = ttk.Frame(self.notebook) self.frameSysInfo = ttk.Frame(self.notebook) self.notebook.add(self.frameSniff, text='Sniffer') self.notebook.add(self.frameArp, text='ARP') self.notebook.add(self.frameDhcp, text='DHCP Servers') self.notebook.add(self.frameSysInfo, text='System Info') self.notebook.pack(fill=BOTH) self.sniffLabelFrame = ttk.LabelFrame(self.frameSniff, text="Sniffer") self.sniffLabelFrame.pack(padx=10, pady=10) self.sniffButton = ttk.Button(self.sniffLabelFrame, text="Sniff", command=self.beginSniff) self.sniffButton.pack(side=LEFT) self.stopSniffButton = ttk.Button(self.sniffLabelFrame, text="Stop sniffing", command=self.stopSniff, state="disabled") self.stopSniffButton.pack(side=LEFT) self.sniffTv = ttk.Treeview(self.frameSniff) ysb = ttk.Scrollbar(self, orient='vertical', command=self.sniffTv.yview) xsb = ttk.Scrollbar(self, orient='horizontal', command=self.sniffTv.xview) self.sniffTv.configure(yscroll=ysb.set, xscroll=xsb.set) self.sniffTv['columns'] = ('senderip', 'sendermac', 'received') self.sniffTv.heading('#0', text='Description', anchor='w') self.sniffTv.column('#0', anchor='w') self.sniffTv.heading('senderip', text='Sender IP') self.sniffTv.column('senderip', width=100) self.sniffTv.heading('sendermac', text='Sender MAC') self.sniffTv.column('sendermac', width=100) self.sniffTv.heading('received', text='Received at') self.sniffTv.column('received', width=100) self.sniffTv.pack(fill=BOTH) self.arpLabel = ttk.Label(self.frameArp, text="ARP cache") self.arpLabel.pack() self.arpTv = ttk.Treeview(self.frameArp) self.arpTv['columns'] = ('ip', 'status') self.arpTv.heading('#0', text='MAC address', anchor='w') self.arpTv.column('#0', anchor='w') self.arpTv.heading('ip', text='IP address') self.arpTv.column('ip', width=100) self.arpTv.heading('status', text='Status') self.arpTv.column('status', width=100) self.arpTv.pack(fill=X) self.addDhcpLabelFrame = ttk.LabelFrame(self.frameDhcp, text="Add trusted server") self.addDhcpLabelFrame.pack(padx=10, pady=10) self.addDhcpNameLabel = ttk.Label(self.addDhcpLabelFrame, text="Server name") self.addDhcpNameLabel.pack() self.addDhcpNameEntry = ttk.Entry(self.addDhcpLabelFrame) self.addDhcpNameEntry.pack() self.addDhcpIpLabel = ttk.Label(self.addDhcpLabelFrame, text="Server IP address") self.addDhcpIpLabel.pack() self.addDhcpIpEntry = ttk.Entry(self.addDhcpLabelFrame) self.addDhcpIpEntry.pack() self.addDhcpMacLabel = ttk.Label(self.addDhcpLabelFrame, text="Server Mac address") self.addDhcpMacLabel.pack() self.addDhcpMacEntry = ttk.Entry(self.addDhcpLabelFrame) self.addDhcpMacEntry.pack() self.addDhcpButton = ttk.Button(self.addDhcpLabelFrame, text="Add", command=self.addDhcpButtonPressed) self.addDhcpButton.pack() self.clrDhcpButton = ttk.Button(self.addDhcpLabelFrame, text="Clear", command=self.clrDhcpButtonPressed) self.clrDhcpButton.pack() self.dhcpTv = ttk.Treeview(self.frameDhcp) self.dhcpTv['columns'] = ('ip', 'mac', 'date') self.dhcpTv.heading('#0', text='Server name', anchor='w') self.dhcpTv.column('#0', anchor='w') self.dhcpTv.heading('ip', text='IP address') self.dhcpTv.column('ip', width=100) self.dhcpTv.heading('mac', text='MAC address') self.dhcpTv.column('mac', width=100) self.dhcpTv.heading('date', text='Date added') self.dhcpTv.column('date', width=100) self.dhcpTv.pack(fill=X) strVersion = 'Python ' + platform.python_version() self.versionLabel = ttk.Label(self.frameSysInfo, text=strVersion) self.versionLabel.pack() strPlatform = 'Platform: ' + platform.platform() self.platformLabel = ttk.Label(self.frameSysInfo, text=strPlatform) self.platformLabel.pack() self.dhcpDefender = DhcpDefender(self) self.arpDefender = ArpDefender(self) self.sniffer = Sniffer(1, 'Sniffer-1', 1, self) self.sniffer.start() self.updater = Updater(1, 'Updater-1', 1, self) self.updater.start() def addDhcpButtonPressed(self): name = self.addDhcpNameEntry.get() ip = self.addDhcpIpEntry.get() mac = self.addDhcpMacEntry.get().lower() isValidIp =re.match(r"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$",ip) macRe = re.compile(r'^([0-9A-F]{1,2})' + '\:([0-9A-F]{1,2})'*5 + '$', re.IGNORECASE) isValidMac = macRe.match(mac) if isValidIp and isValidMac: self.dhcpDefender.add_trusted_server(name, ip, mac) def clrDhcpButtonPressed(self): self.dhcpDefender.clear_db() def beginSniff(self): self.sniffButton.config(state="disabled") self.stopSniffButton.config(state="normal") self.sniffer.resume() def stopSniff(self): self.stopSniffButton.config(state="disabled") self.sniffButton.config(state="normal") self.sniffer.pause() def updateSniffTv(self, packet): if packet[ARP].op == 1: response = 'Request: ' + packet[ARP].psrc + ' is asking about ' + packet[ARP].pdst elif packet[ARP].op == 2: response = 'Response: ' + packet[ARP].hwsrc + ' has address ' + packet[ARP].psrc timeStr = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") self.sniffTv.insert("", 0, text=response, values=(packet[ARP].psrc, packet[ARP].hwsrc, timeStr)) def updateDhcpTv(self, trusted_servers): self.dhcpTv.delete(*self.dhcpTv.get_children()) for server in trusted_servers: self.dhcpTv.insert("", 0, text=server['name'], values=(server['ip'], server['mac'], server['date'])) def import_arp_cache(self): self.arpTv.delete(*self.arpTv.get_children()) out, err = Popen(['arp', '-na'], stdout=PIPE, stderr=PIPE).communicate() out = out.splitlines() for line in out: ip = self.find_between(line, '(', ')') mac = self.find_between(line, 'at ', ' on') self.arpTv.insert("", 0, text=mac, values=(ip, "")) def find_between(self, s, first, last): try: start = s.index(first) + len(first) end = s.index(last, start) return s[start:end] except ValueError: return "" def process_arp(self, packet): result = self.arpDefender.arp_pkt_callback(packet) if result == -1: # inconsistent headers print("inconsistent headers") self.statusbar.config(text="Warning: Inconsistent header packet detected.") pass elif result == 0: # failed active check print("failed active check") self.statusbar.config(text="Warning: You might be under ARP spoof attack.") pass elif result == 1: # passed active check print("passed active check") pass def process_dhcp(self, packet): result = self.dhcpDefender.dhcp_pkt_callback(packet) if result == 0: # unknown dhcp server print("Unknown dhcp server") self.statusbar.config(text="Warning: Untrusted DHCP server present in network.") pass elif result == 1: print("Trusted dhcp server") pass
class UpdaterGUI(Window): def __init__( self, parent): self.updater = Updater() super(UpdaterGUI, self).__init__(parent) self.m_dirPicker1.SetPath(self.updater.skin_folder) def UpdateButton( self, event ): result = False if self.updater.skin_folder == "": dlg = wx.MessageDialog( self, "The Steam folder has not been specified. Please input the Steam folder location first.", "Input Steam Folder Location", wx.OK) dlg.ShowModal() dlg.Destroy() event.Skip() if self.updater.current_version == self.updater.latest_version: dlg = wx.MessageDialog( self, "You seem to have the latest version installed already. Do you want to update anyway?", "Latest Version Already Installed", wx.OK | wx.CANCEL) answer = dlg.ShowModal() # Show it dlg.Destroy() # finally destroy it when finished. if answer == wx.ID_OK: result = self.updater.UpdateSkin() else: result = self.updater.UpdateSkin() if result: dlg = wx.MessageDialog( self, "Updated to {}. Please restart Steam.".format(self.updater.latest_version), "Finished", wx.OK) dlg.ShowModal() dlg.Destroy() self.m_staticText2.SetLabel('Installed Version: {}'.format(self.updater.current_version)) def OnClose( self, event ): self.updater.SaveConfig() event.Skip() def UpdateSteamFolder( self, event ): print "cool" print self.m_dirPicker1.GetPath() if not self.updater.SetSteamFolder(self.m_dirPicker1.GetPath()): dlg = wx.MessageDialog( self, "Couldn't find the folder \"{}\". Please double check this is the where steam is installed.".format(self.m_dirPicker1.GetPath()), "Cannot find Steam Folder", wx.OK) dlg.ShowModal() dlg.Destroy() else: self.m_dirPicker1.SetPath(self.updater.skin_folder) event.Skip() def UpdateCurrentVersion( self, event ): self.m_staticText1.SetLabel('Latest Version: {}'.format(self.updater.latest_version)) event.Skip() def UpdateLatestVersion( self, event ): self.m_staticText2.SetLabel('Installed Version: {}'.format(self.updater.current_version)) print self.updater.current_version event.Skip() def UpdateStatusBar( self, event ): self.SetStatusText("Status") event.Skip()
def load(lstm_opts=None): if not lstm_opts: lstm_opts = {"class": "lstm2"} lstm_opts = lstm_opts.copy() lstm_opts.update({"n_out": 10, "from": "in"}) num_inputs = 9 num_outputs = 2 net_topo = { "in": { "class": "dump", "filename": "in" }, "lstm": lstm_opts, "lstm_dump": { "class": "dump", "from": "lstm", "filename": "lstm" }, "output": { "class": "softmax", "loss": "ce", "from": "lstm_dump" } } collected_data = {} DumpLayer.global_debug_container = collected_data net = Network.LayerNetwork.from_json(json_content=net_topo, n_in=num_inputs, n_out={"classes": (num_outputs, 1)}, train_flag=True) net.declare_train_params() # Init dataset and prepare one minibatch. epoch = 1 dataset = Task12AXDataset(num_seqs=1000, seq_ordering="random", chunking="200:200") dataset.init_seq_order(epoch=epoch) batch_gen = dataset.generate_batches(recurrent_net=net.recurrent, batch_size=5000, max_seqs=10) batches = batch_gen.peek_next_n(1) # We need the DummyDevice for assign_dev_data. dev = DummyDevice() assign_success, _ = EngineUtil.assign_dev_data(device=dev, dataset=dataset, batches=batches) assert assign_success dev.initialize(net) dev.update_data() givens = [(net.y[k], dev.y[k]) for k in dev.used_data_keys] givens += [(net.j[k], dev.j[k]) for k in dev.used_data_keys] # Now gradients, updates and compile everything. gradients = { p: T.grad(net.get_objective(), p, known_grads=net.known_grads) for p in net.train_params_vars } updater = Updater(adam=True) updater.initVars(net, gradients) updater.setLearningRate(learning_rate=0.01) trainer = theano.function(inputs=[], outputs=[net.total_cost], givens=givens, updates=updater.getUpdateList(), on_unused_input='warn', name="train_and_updater") for p in net.train_params_vars: collected_data["param:%s" % p.name] = p.get_value() # And finally, run it. cost = trainer() collected_data["cost"] = cost return collected_data
import sys from Updater import Updater repo_name_src = 'gaia0531ta1' repo_name_dst = 'gaia0531ta2' endpoint_src = 'http://gaiadev01.isi.edu:7200/repositories/' + repo_name_src endpoint_dst = 'http://gaiadev01.isi.edu:7200/repositories/' + repo_name_dst output = '/Users/jenniferchen/Documents/AIDA/ta2pipline/store_data/' + repo_name_dst graph = 'http://www.isi.edu/baseline-20190605-001' has_jl = True print('---') print('your source endpoint: ', endpoint_src) print('your destination endpoint: ', endpoint_dst) print('your output: ', output) print('your graph: ', graph) print('your has jl: ', has_jl) print('---') up = Updater(endpoint_src, endpoint_dst, repo_name_src, output, graph, has_jl) #up.run_delete_ori() # run this only if creating the first named graph in the repo up.run_load_jl(entity_clusters='clusters-baseline-20190605-001.jl') up.run_entity_nt() up.run_event_nt() up.run_relation_nt() up.run_insert_proto() up.run_super_edge()
def load(lstm_opts=None): if not lstm_opts: lstm_opts = {"class": "lstm2"} lstm_opts = lstm_opts.copy() lstm_opts.update({"n_out": 10, "from": "in"}) num_inputs = 9 num_outputs = 2 net_topo = { "in": {"class": "dump", "filename": "in"}, "lstm": lstm_opts, "lstm_dump": {"class": "dump", "from": "lstm", "filename": "lstm"}, "output": {"class": "softmax", "loss": "ce", "from": "lstm_dump"} } collected_data = {} DumpLayer.global_debug_container = collected_data net = Network.LayerNetwork.from_json( json_content=net_topo, n_in=num_inputs, n_out={"classes": (num_outputs, 1)}, train_flag=True ) net.declare_train_params() # Init dataset and prepare one minibatch. epoch = 1 dataset = Task12AXDataset(num_seqs=1000, seq_ordering="random", chunking="200:200") dataset.init_seq_order(epoch=epoch) batch_gen = dataset.generate_batches( recurrent_net=net.recurrent, batch_size=5000, max_seqs=10) batches = batch_gen.peek_next_n(1) # We need the DummyDevice for assign_dev_data. dev = DummyDevice() assign_success, _ = EngineUtil.assign_dev_data(device=dev, dataset=dataset, batches=batches) assert assign_success dev.initialize(net) dev.update_data() givens = [(net.y[k], dev.y[k]) for k in dev.used_data_keys] givens += [(net.j[k], dev.j[k]) for k in dev.used_data_keys] # Now gradients, updates and compile everything. gradients = {p: T.grad(net.get_objective(), p, known_grads=net.known_grads) for p in net.train_params_vars} updater = Updater(adam=True) updater.initVars(net, gradients) updater.setLearningRate(learning_rate=0.01) trainer = theano.function( inputs=[], outputs=[net.total_cost], givens=givens, updates=updater.getUpdateList(), on_unused_input='warn', name="train_and_updater") for p in net.train_params_vars: collected_data["param:%s" % p.name] = p.get_value() # And finally, run it. cost = trainer() collected_data["cost"] = cost return collected_data
def __init__( self, parent): self.updater = Updater() super(UpdaterGUI, self).__init__(parent) self.m_dirPicker1.SetPath(self.updater.skin_folder)
def __init__(self): Updater.__init__(self)
detector_model_path = 'models/model_frcnn.hdf5' model_rpn.load_weights(detector_model_path, by_name=True) model_classifier.load_weights(detector_model_path, by_name=True) # COMPILE MODEL model_rpn.compile(optimizer='sgd', loss='mse') model_classifier.compile(optimizer='sgd', loss='mse') # LOAD PREDICTION MODEL classifier_model_path = 'models/MobileNetV2_192_NoCar.h5' classifier = load_model(classifier_model_path) # SETUP BOT AND FOLDERS alphabeth = [ 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'Z' ] if not os.path.exists("res"): os.makedirs("res") classif_input = 192 marker_true = cv2.imread("V_tick.png", -1) marker_false = cv2.imread("X_tick.png", -1) bot_id = '*********' updater = Updater(bot_id) updater.setPhotoHandler(imageHandler) print('---------- BOT IS READY') updater.start()
class UpdaterWidget(QtWidgets.QWidget, Ui_updater_gui): cancel_signal = QtCore.pyqtSignal() def __init__(self): """ The GUI window that runs the updater in a separate thread. Used mainly to show the progress of the updater and ensure the user that it is doing something while it downloads the newest version from github """ super().__init__() self.setupUi(self) self.settings = QtCore.QSettings('SomeGuySoftware', 'dfr_updater') self.dfr_settings = QtCore.QSettings('SomeGuySoftware', 'RedditDownloader') self.restoreGeometry( self.settings.value('geometry', self.saveGeometry())) self.download_url = self.settings.value('download_url', None, type=str) self.download_name = self.settings.value('download_name', None, type=str) self.new_version = self.settings.value('new_version', None, type=str) self.launch_checkbox.setChecked( self.settings.value('launch_checkbox', False, type=bool)) self.save_height = True self.label.setOpenExternalLinks(True) self.program_files_location = self.settings.value( 'program_files_location', None, type=str) self.running = False self.up_to_date = False self.button_box.accepted.connect(self.assign_accept) self.button_box.rejected.connect(self.assign_reject) self.progress_bar.setVisible(False) self.launch_checkbox.setVisible(False) def run(self): """Moves the Updater module to a separate thread and runs it""" self.update_label('Starting updater...') self.running_gui_shift() self.thread = QtCore.QThread() self.updater = Updater(self.download_url, self.download_name, self.new_version, self.program_files_location) self.cancel_signal.connect(self.updater.stop) self.updater.moveToThread(self.thread) self.thread.started.connect(self.updater.run_update) self.updater.update_label.connect(self.update_label) self.updater.setup_progress_bar.connect(self.setup_progress_bar) self.updater.update_progress_bar.connect(self.update_progress_bar) self.updater.error_signal.connect(self.update_error) self.updater.finished.connect(self.thread.quit) self.updater.finished.connect(self.updater.deleteLater) self.updater.finished.connect(self.finished_gui_shift) self.thread.finished.connect(self.thread.deleteLater) self.thread.start() def assign_accept(self): if self.up_to_date: self.open_file_location() else: self.run() def assign_reject(self): if self.running: self.stop_download() else: self.close() def running_gui_shift(self): """Changes some GUI behavior so as not to interfere with the updater while providing a way to stop it""" self.progress_bar.setVisible(True) self.running = True self.button_box.button(QtWidgets.QDialogButtonBox.Ok).setVisible(False) self.button_box.button( QtWidgets.QDialogButtonBox.Cancel).setText('Stop') def finished_gui_shift(self): """Changes the GUI behaviour again so that certain operations can be performed once the update is complete""" self.running = False self.up_to_date = True self.update_label( 'Update complete. You are now running the latest version.') self.button_box.button(QtWidgets.QDialogButtonBox.Ok).setVisible(True) self.button_box.button( QtWidgets.QDialogButtonBox.Ok).setText('Open exe location') self.button_box.button( QtWidgets.QDialogButtonBox.Cancel).setText('Close') self.launch_checkbox.setVisible(True) self.dfr_settings.setValue('first_run', True) def stopped_gui_shift(self): """A different GUI shift that provides a way to restart the updater if it is stopped""" self.running = False self.button_box.button(QtWidgets.QDialogButtonBox.Ok).setVisible(True) self.button_box.button( QtWidgets.QDialogButtonBox.Cancel).setText('Close') def update_label(self, text): self.label.setText(text) def setup_progress_bar(self, setup): if setup > 100: minimum = 0 - (setup - 100) maximum = 100 else: minimum = 0 maximum = setup self.progress_bar.setMinimum(minimum) self.progress_bar.setMaximum(maximum) self.progress_bar.setValue(minimum) def update_progress_bar(self, length): self.progress_bar.setValue(self.progress_bar.value() + length) def stop_download(self): self.cancel_signal.emit() self.finished_gui_shift() def open_file_location(self): """Opens the source folder of the new program version""" if sys.platform == 'win32': os.startfile(self.program_files_location) else: subprocess.call(['xdg-open', self.program_files_location]) def launch_program(self): """Launches the newly installed program version""" if sys.platform == 'win32': os.startfile( os.path.join(self.program_files_location, 'DownloaderForReddit.exe')) elif sys.platform == 'linux': subprocess.Popen([ os.path.join(self.program_files_location, 'DownloaderForReddit'), os.path.join(self.program_files_location, 'DownloaderForReddit') ]) """ The methods below are to alert the user to any type of error that may arise during the update process and provide them with a way to resolve the error manually """ def update_error(self, code): self.stopped_gui_shift() if code[0] == 0: self.update_label( 'There was a problem establishing a connection to the github download url. Please try ' 'the update again. If the problem persists the package can be downloaded manually at: ' '<a href="https://github.com/MalloyDelacroix/DownloaderForReddit/releases">https://github.com/MalloyDelacroix/DownloaderForReddit/releases</a>' ) self.setMinimumHeight(self.height() + self.label.height()) self.save_height = False elif code[0] == 1: self.update_label( 'There was a problem deleting the outdated files. There was no Downloader for Reddit ' 'executable found in the folder location. Please try the update again. If the ' 'problem persists, the latest update has been downloaded at can be moved manually to ' 'the desired location.') self.setMinimumHeight(self.height() + self.label.height()) self.button_box.button(QtWidgets.QDialogButtonBox.Ok).setText( 'Open Download Location') self.button_box.accepted.connect( lambda: self.open_temporary_download_location(code[1])) self.save_height = False elif code[0] == 2: self.update_label( 'There was a problem extracting or moving the downloaded files to the programs directory ' 'The original program files have been removed. You may manually move the downloaded ' 'files to the desired location and run the Downloader for Reddit from there.' ) self.setMinimumHeight(self.height() + self.label.height()) self.button_box.button(QtWidgets.QDialogButtonBox.Ok).setText( 'Open Download Location') self.button_box.accepted.connect( lambda: self.open_temporary_download_location(code[1])) self.save_height = False def open_temporary_download_location(self, location): """ In case the downloader fails after downloading the new update but before moving it, this will open the temporary location """ if sys.platform == 'win32': os.startfile(location) else: subprocess.call(['xdg-open', location]) self.close() def closeEvent(self, QCloseEvent): if self.save_height: self.settings.setValue('geometry', self.saveGeometry()) self.settings.setValue('launch_checkbox', self.launch_checkbox.isChecked()) if self.launch_checkbox.isChecked() and self.launch_checkbox.isVisible( ): self.launch_program()
def run(args): config = args[0] slapgrid = None supervisord_pid_file = os.path.join(config["instance_root"], "var", "run", "supervisord.pid") subprocess.check_call([config["git_binary"], "config", "--global", "http.sslVerify", "false"]) previous_revision = None run_software = True # Write our own software.cfg to use the local repository custom_profile_path = os.path.join(config["working_directory"], "software.cfg") config["custom_profile_path"] = custom_profile_path vcs_repository_list = config["vcs_repository_list"] profile_content = None assert len(vcs_repository_list), "we must have at least one repository" for vcs_repository in vcs_repository_list: url = vcs_repository["url"] buildout_section_id = vcs_repository.get("buildout_section_id", None) repository_id = buildout_section_id or url.split("/")[-1].split(".")[0] repository_path = os.path.join(config["working_directory"], repository_id) vcs_repository["repository_id"] = repository_id vcs_repository["repository_path"] = repository_path if profile_content is None: profile_content = """ [buildout] extends = %(software_config_path)s """ % { "software_config_path": os.path.join(repository_path, config["profile_path"]) } if not (buildout_section_id is None): profile_content += """ [%(buildout_section_id)s] repository = %(repository_path)s branch = %(branch)s """ % { "buildout_section_id": buildout_section_id, "repository_path": repository_path, "branch": vcs_repository.get("branch", "master"), } custom_profile = open(custom_profile_path, "w") custom_profile.write(profile_content) custom_profile.close() config["repository_path"] = repository_path sys.path.append(repository_path) test_suite_title = config["test_suite_title"] or config["test_suite"] retry_software = False try: while True: # kill processes from previous loop if any try: for pgpid in process_group_pid_set: try: os.killpg(pgpid, signal.SIGTERM) except: pass process_group_pid_set.clear() full_revision_list = [] # Make sure we have local repository for vcs_repository in vcs_repository_list: repository_path = vcs_repository["repository_path"] repository_id = vcs_repository["repository_id"] if not os.path.exists(repository_path): parameter_list = [config["git_binary"], "clone", vcs_repository["url"]] if vcs_repository.get("branch") is not None: parameter_list.extend(["-b", vcs_repository.get("branch")]) parameter_list.append(repository_path) subprocess.check_call(parameter_list) # Make sure we have local repository updater = Updater(repository_path, git_binary=config["git_binary"]) updater.checkout() revision = "-".join(updater.getRevision()) full_revision_list.append("%s=%s" % (repository_id, revision)) revision = ",".join(full_revision_list) if previous_revision == revision: time.sleep(120) if not (retry_software): continue retry_software = False previous_revision = revision print config portal_url = config["test_suite_master_url"] test_result_path = None test_result = (test_result_path, revision) if portal_url: if portal_url[-1] != "/": portal_url += "/" portal = xmlrpclib.ServerProxy("%s%s" % (portal_url, "portal_task_distribution"), allow_none=1) master = portal.portal_task_distribution assert master.getProtocolRevision() == 1 test_result = safeRpcCall( master.createTestResult, config["test_suite"], revision, [], False, test_suite_title, config["test_node_title"], config["project_title"], ) print "testnode, test_result : %r" % (test_result,) if test_result: test_result_path, test_revision = test_result if revision != test_revision: for i, repository_revision in enumerate(test_revision.split(",")): vcs_repository = vcs_repository_list[i] repository_path = vcs_repository["repository_path"] # other testnodes on other boxes are already ready to test another # revision updater = Updater( repository_path, git_binary=config["git_binary"], revision=repository_revision.split("-")[1], ) updater.checkout() # Now prepare the installation of SlapOS and create instance slapos_controler = SlapOSControler(config, process_group_pid_set=process_group_pid_set) for method_name in ("runSoftwareRelease", "runComputerPartition"): stdout, stderr = getInputOutputFileList(config, method_name) slapos_method = getattr(slapos_controler, method_name) status_dict = slapos_method( config, environment=config["environment"], process_group_pid_set=process_group_pid_set, stdout=stdout, stderr=stderr, ) if status_dict["status_code"] != 0: break if status_dict["status_code"] != 0: safeRpcCall(master.reportTaskFailure, test_result_path, status_dict, config["test_node_title"]) retry_software = True continue partition_path = os.path.join(config["instance_root"], config["partition_reference"]) run_test_suite_path = os.path.join(partition_path, "bin", "runTestSuite") if not os.path.exists(run_test_suite_path): raise ValueError("No %r provided" % run_test_suite_path) run_test_suite_revision = revision if isinstance(revision, tuple): revision = ",".join(revision) # Deal with Shebang size limitation file_object = open(run_test_suite_path, "r") line = file_object.readline() file_object.close() invocation_list = [] if line[:2] == "#!": invocation_list = line[2:].split() invocation_list.extend( [ run_test_suite_path, "--test_suite", config["test_suite"], "--revision", revision, "--test_suite_title", test_suite_title, "--node_quantity", config["node_quantity"], "--master_url", config["test_suite_master_url"], ] ) run_test_suite = subprocess.Popen(invocation_list) process_group_pid_set.add(run_test_suite.pid) run_test_suite.wait() process_group_pid_set.remove(run_test_suite.pid) except SubprocessError: time.sleep(120) continue finally: # Nice way to kill *everything* generated by run process -- process # groups working only in POSIX compilant systems # Exceptions are swallowed during cleanup phase print "going to kill %r" % (process_group_pid_set,) for pgpid in process_group_pid_set: try: os.killpg(pgpid, signal.SIGTERM) except: pass try: if os.path.exists(supervisord_pid_file): os.kill(int(open(supervisord_pid_file).read().strip()), signal.SIGTERM) except: pass
class EmailClient: def __init__(self): self.updater_client = Updater("Urls.config") self.email_content = self.set_up_email_content() # set up the SMTP server self.email, self.password = self.get_creditionals( "Creditionals.config") self.server = smtplib.SMTP("smtp-mail.outlook.com", 587) #self.server = smtplib.SMTP("smtp.gmail.com", 587) self.server.starttls() self.server.login(self.email, self.password) def send_email(self, sender_email, single=True): # the single parameter is used in order to dictate if the connection is closed or not. self.server.sendmail(self.email, sender_email, self.email_content) if (single): self.close_connection() def get_emails(self): file_object = open("Emails.config", "r") config_by_line = file_object.readlines() config_by_line = [line.rstrip() for line in config_by_line] file_object.close() return config_by_line def send_email_group(self): # would be easy to implement; simply just read from a text file of some sort... Then move server.quit to here email_list = self.get_emails() for email in email_list: print "Sending email to {}...".format(email) self.send_email(email, single=False) self.close_connection() def set_up_email_content(self): updated_list = self.updater_client.get_updated_blog_list() #updated_list = ["https://krebsonsecurity.com","https://maxwelldulin.com/blog"] msg = """ Hey! Security blogs are amazing! So, here's your custom list of followed blogs that have been updated in the last day: """ blogs_updated = "" for blog in updated_list: blogs_updated += '\t' + blog + '\n' if (blogs_updated == ""): msg += "No blogs you are following have been updated. :( " else: msg += blogs_updated msg += "\n Have a wonderful rest of your day; and happy reading!\n" msg += "This blog follower list was created by Maxwell Dulin at http://maxwelldulin.com" subject = "Security Blog Post Update" message = "Subject: {}\n\n{}".format(subject, msg) return message def close_connection(self): self.server.quit() def get_creditionals(self, filename): file_object = open(filename, "r") files_list_per_line = file_object.readlines() files_list_per_line = list( filter(lambda x: x != '\n', files_list_per_line)) formatted_file = list(map(lambda x: x.rstrip(), files_list_per_line)) username = formatted_file[0].replace(" ", "").split(":")[1] password = formatted_file[1].strip(" ").replace(" ", "").split(":")[1] return username, password
def on_request(request_id, topic, data): Logger().write(LogVerbosity.Debug, "Master request: " + topic) if topic == "get_last_version": SlaveClientController.slave_ns.emit("response", request_id, Updater().check_version(), Updater().last_version)
def run(self): log = self.log process_manager = self.process_manager config = self.config slapgrid = None previous_revision = None run_software = True # Write our own software.cfg to use the local repository custom_profile_path = os.path.join(config['working_directory'], 'software.cfg') config['custom_profile_path'] = custom_profile_path vcs_repository_list = config['vcs_repository_list'] profile_content = '' assert len(vcs_repository_list), "we must have at least one repository" try: # BBB: Accept global profile_path, which is the same as setting it for the # first configured repository. profile_path = config.pop(PROFILE_PATH_KEY) except KeyError: pass else: vcs_repository_list[0][PROFILE_PATH_KEY] = profile_path profile_path_count = 0 for vcs_repository in vcs_repository_list: url = vcs_repository['url'] buildout_section_id = vcs_repository.get('buildout_section_id', None) repository_id = buildout_section_id or \ url.split('/')[-1].split('.')[0] repository_path = os.path.join(config['working_directory'],repository_id) vcs_repository['repository_id'] = repository_id vcs_repository['repository_path'] = repository_path try: profile_path = vcs_repository[PROFILE_PATH_KEY] except KeyError: pass else: profile_path_count += 1 if profile_path_count > 1: raise ValueError(PROFILE_PATH_KEY + ' defined more than once') profile_content = """ [buildout] extends = %(software_config_path)s """ % {'software_config_path': os.path.join(repository_path, profile_path)} if not(buildout_section_id is None): profile_content += """ [%(buildout_section_id)s] repository = %(repository_path)s branch = %(branch)s """ % {'buildout_section_id': buildout_section_id, 'repository_path' : repository_path, 'branch' : vcs_repository.get('branch','master')} if not profile_path_count: raise ValueError(PROFILE_PATH_KEY + ' not defined') custom_profile = open(custom_profile_path, 'w') custom_profile.write(profile_content) custom_profile.close() config['repository_path'] = repository_path sys.path.append(repository_path) test_suite_title = config['test_suite_title'] or config['test_suite'] retry = False retry_software_count = 0 same_revision_count = 0 try: while True: remote_test_result_needs_cleanup = False remote_logger = None remote_logger_thread = None try: # kill processes from previous loop if any process_manager.killPreviousRun() full_revision_list = [] # Make sure we have local repository for vcs_repository in vcs_repository_list: repository_path = vcs_repository['repository_path'] repository_id = vcs_repository['repository_id'] if not os.path.exists(repository_path): parameter_list = [config['git_binary'], 'clone', vcs_repository['url']] if vcs_repository.get('branch') is not None: parameter_list.extend(['-b',vcs_repository.get('branch')]) parameter_list.append(repository_path) log(subprocess.check_output(parameter_list, stderr=subprocess.STDOUT)) # Make sure we have local repository updater = Updater(repository_path, git_binary=config['git_binary'], log=log, process_manager=process_manager) updater.checkout() revision = "-".join(updater.getRevision()) full_revision_list.append('%s=%s' % (repository_id, revision)) revision = ','.join(full_revision_list) if previous_revision == revision: log('Same Revision') same_revision_count += 1 if not(retry) and same_revision_count <= 2: log('Sleeping a bit since same revision') time.sleep(DEFAULT_SLEEP_TIMEOUT) continue same_revision_count = 0 log('Retrying install or checking if previous test was cancelled') retry = False previous_revision = revision portal_url = config['test_suite_master_url'] test_result_path = None test_result = (test_result_path, revision) if portal_url: if portal_url[-1] != '/': portal_url += '/' portal = xmlrpclib.ServerProxy("%s%s" % (portal_url, 'portal_task_distribution'), allow_none=1) assert safeRpcCall(log, portal, "getProtocolRevision", True) == 1 test_result = safeRpcCall(log, portal, "createTestResult", True, config['test_suite'], revision, [], False, test_suite_title, config['test_node_title'], config['project_title']) remote_test_result_needs_cleanup = True log("testnode, test_result : %r" % (test_result, )) if test_result: test_result_path, test_revision = test_result if config.get('log_file'): remote_logger = RemoteLogger(log, config['log_file'], config['test_node_title'], process_manager) remote_logger.portal = portal remote_logger.test_result_path = test_result_path remote_logger_thread = threading.Thread(target=remote_logger) remote_logger_thread.start() if revision != test_revision: previous_revision = test_revision log('Disagreement on tested revision, checking out:') for i, repository_revision in enumerate(test_revision.split(',')): vcs_repository = vcs_repository_list[i] repository_path = vcs_repository['repository_path'] revision = repository_revision.rsplit('-', 1)[1] # other testnodes on other boxes are already ready to test another # revision log(' %s at %s' % (repository_path, revision)) updater = Updater(repository_path, git_binary=config['git_binary'], revision=revision, log=log, process_manager=process_manager) updater.checkout() # Now prepare the installation of SlapOS and create instance slapproxy_log = os.path.join(config['log_directory'], 'slapproxy.log') log('Configured slapproxy log to %r' % slapproxy_log) log('testnode, retry_software_count : %r' % retry_software_count) slapos_controler = SlapOSControler.SlapOSControler(config, log=log, slapproxy_log=slapproxy_log, process_manager=process_manager, reset_software=(retry_software_count>0 and retry_software_count%10 == 0)) for method_name in ("runSoftwareRelease", "runComputerPartition",): slapos_method = getattr(slapos_controler, method_name) status_dict = slapos_method(config, environment=config['environment'], ) if status_dict['status_code'] != 0: retry = True retry_software_count += 1 raise SubprocessError(status_dict) else: retry_software_count = 0 # Give some time so computer partitions may start # as partitions can be of any kind we have and likely will never have # a reliable way to check if they are up or not ... time.sleep(20) run_test_suite_path_list = glob.glob("%s/*/bin/runTestSuite" %config['instance_root']) if not len(run_test_suite_path_list): raise ValueError('No runTestSuite provided in installed partitions.') run_test_suite_path = run_test_suite_path_list[0] run_test_suite_revision = revision if isinstance(revision, tuple): revision = ','.join(revision) # Deal with Shebang size limitation line = open(run_test_suite_path, 'r').readline() invocation_list = [] if line[:2] == '#!': invocation_list = line[2:].split() invocation_list.extend([run_test_suite_path, '--test_suite', config['test_suite'], '--revision', revision, '--test_suite_title', test_suite_title, '--node_quantity', config['node_quantity'], '--master_url', portal_url]) bt5_path_list = config.get("bt5_path") if bt5_path_list not in ('', None,): invocation_list.extend(["--bt5_path", bt5_path_list]) # From this point, test runner becomes responsible for updating test # result. We only do cleanup if the test runner itself is not able # to run. process_manager.spawn(*invocation_list, cwd=config['test_suite_directory'], log_prefix='runTestSuite', get_output=False) if remote_logger: remote_logger.quit = True remote_logger_thread.join() except SubprocessError, e: log("SubprocessError", exc_info=sys.exc_info()) if remote_logger: remote_logger.finish = True remote_logger_thread.join() if remote_test_result_needs_cleanup: safeRpcCall(log, portal, "reportTaskFailure", True, test_result_path, e.status_dict, config['test_node_title']) log("SubprocessError, going to sleep %s" % DEFAULT_SLEEP_TIMEOUT) time.sleep(DEFAULT_SLEEP_TIMEOUT) continue except CancellationError, e: log("CancellationError", exc_info=sys.exc_info()) process_manager.under_cancellation = False retry = True continue
def run(config): log = config['logger'] slapgrid = None global supervisord_pid_file supervisord_pid_file = os.path.join(config['instance_root'], 'var', 'run', 'supervisord.pid') previous_revision = None run_software = True # Write our own software.cfg to use the local repository custom_profile_path = os.path.join(config['working_directory'], 'software.cfg') config['custom_profile_path'] = custom_profile_path vcs_repository_list = config['vcs_repository_list'] profile_content = None assert len(vcs_repository_list), "we must have at least one repository" try: # BBB: Accept global profile_path, which is the same as setting it for the # first configured repository. profile_path = config.pop(PROFILE_PATH_KEY) except KeyError: pass else: vcs_repository_list[0][PROFILE_PATH_KEY] = profile_path for vcs_repository in vcs_repository_list: url = vcs_repository['url'] buildout_section_id = vcs_repository.get('buildout_section_id', None) repository_id = buildout_section_id or \ url.split('/')[-1].split('.')[0] repository_path = os.path.join(config['working_directory'],repository_id) vcs_repository['repository_id'] = repository_id vcs_repository['repository_path'] = repository_path try: profile_path = vcs_repository[PROFILE_PATH_KEY] except KeyError: pass else: if profile_content is not None: raise ValueError(PROFILE_PATH_KEY + ' defined more than once') profile_content = """ [buildout] extends = %(software_config_path)s """ % {'software_config_path': os.path.join(repository_path, profile_path)} if not(buildout_section_id is None): profile_content += """ [%(buildout_section_id)s] repository = %(repository_path)s branch = %(branch)s """ % {'buildout_section_id': buildout_section_id, 'repository_path' : repository_path, 'branch' : vcs_repository.get('branch','master')} if profile_content is None: raise ValueError(PROFILE_PATH_KEY + ' not defined') custom_profile = open(custom_profile_path, 'w') custom_profile.write(profile_content) custom_profile.close() config['repository_path'] = repository_path sys.path.append(repository_path) test_suite_title = config['test_suite_title'] or config['test_suite'] retry_software = False try: while True: remote_test_result_needs_cleanup = False # kill processes from previous loop if any try: killPreviousRun() process_group_pid_set.clear() full_revision_list = [] # Make sure we have local repository for vcs_repository in vcs_repository_list: repository_path = vcs_repository['repository_path'] repository_id = vcs_repository['repository_id'] if not os.path.exists(repository_path): parameter_list = [config['git_binary'], 'clone', vcs_repository['url']] if vcs_repository.get('branch') is not None: parameter_list.extend(['-b',vcs_repository.get('branch')]) parameter_list.append(repository_path) log(subprocess.check_output(parameter_list, stderr=subprocess.STDOUT)) # Make sure we have local repository updater = Updater(repository_path, git_binary=config['git_binary'], log=log, realtime_output=False) updater.checkout() revision = "-".join(updater.getRevision()) full_revision_list.append('%s=%s' % (repository_id, revision)) revision = ','.join(full_revision_list) if previous_revision == revision: log('Sleeping a bit') time.sleep(120) if not(retry_software): continue log('Retrying install') retry_software = False previous_revision = revision portal_url = config['test_suite_master_url'] test_result_path = None test_result = (test_result_path, revision) if portal_url: if portal_url[-1] != '/': portal_url += '/' portal = xmlrpclib.ServerProxy("%s%s" % (portal_url, 'portal_task_distribution'), allow_none=1) master = portal.portal_task_distribution assert safeRpcCall(master.getProtocolRevision) == 1 test_result = safeRpcCall(master.createTestResult, config['test_suite'], revision, [], False, test_suite_title, config['test_node_title'], config['project_title']) remote_test_result_needs_cleanup = True log("testnode, test_result : %r" % (test_result, )) if test_result: test_result_path, test_revision = test_result if revision != test_revision: log('Disagreement on tested revision, checking out:') for i, repository_revision in enumerate(test_revision.split(',')): vcs_repository = vcs_repository_list[i] repository_path = vcs_repository['repository_path'] revision = repository_revision.split('-')[1] # other testnodes on other boxes are already ready to test another # revision log(' %s at %s' % (repository_path, revision)) updater = Updater(repository_path, git_binary=config['git_binary'], revision=revision, log=log, realtime_output=False) updater.checkout() # Now prepare the installation of SlapOS and create instance slapproxy_log = os.path.join(config['log_directory'], 'slapproxy.log') log('Configured slapproxy log to %r' % slapproxy_log) slapos_controler = SlapOSControler.SlapOSControler(config, process_group_pid_set=process_group_pid_set, log=log, slapproxy_log=slapproxy_log) for method_name in ("runSoftwareRelease", "runComputerPartition"): stdout, stderr = getInputOutputFileList(config, method_name) slapos_method = getattr(slapos_controler, method_name) status_dict = slapos_method(config, environment=config['environment'], process_group_pid_set=process_group_pid_set, stdout=stdout, stderr=stderr ) if status_dict['status_code'] != 0: retry_software = True raise SubprocessError(status_dict) run_test_suite_path = config['runTestSuite'] if not os.path.exists(run_test_suite_path): raise SubprocessError({ 'command': 'os.path.exists(run_test_suite_path)', 'status_code': 1, 'stdout': '', 'stderr': 'File does not exist: %r' % (run_test_suite_path, ), }) run_test_suite_revision = revision if isinstance(revision, tuple): revision = ','.join(revision) # Deal with Shebang size limitation line = open(run_test_suite_path, 'r').readline() invocation_list = [] if line[:2] == '#!': invocation_list = line[2:].split() invocation_list.extend([run_test_suite_path, '--test_suite', config['test_suite'], '--revision', revision, '--test_suite_title', test_suite_title, '--node_quantity', config['node_quantity'], '--master_url', config['test_suite_master_url']]) # From this point, test runner becomes responsible for updating test # result. # XXX: is it good for all cases (eg: test runner fails too early for # any custom code to pick the failure up and react ?) remote_test_result_needs_cleanup = False run_test_suite = subprocess.Popen(invocation_list, preexec_fn=os.setsid, cwd=config['test_suite_directory'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) process_group_pid_set.add(run_test_suite.pid) log(run_test_suite.communicate()[0]) process_group_pid_set.remove(run_test_suite.pid) except SubprocessError, e: if remote_test_result_needs_cleanup: safeRpcCall(master.reportTaskFailure, test_result_path, e.status_dict, config['test_node_title']) time.sleep(120) continue finally: # Nice way to kill *everything* generated by run process -- process # groups working only in POSIX compilant systems # Exceptions are swallowed during cleanup phase killPreviousRun()
import sys from Updater import Updater endpoint = input('Enter localhost with or without / (no route name like "update", "query")\n') output = input('Enter output folder with / \n') graph = input('Enter graph name \n') has_jl = input('Enter "True" if you has jl, otherwise press enter\n') print('---') print('you endpoint: ', endpoint) print('you output: ', output) print('you graph: ', graph) print('you has jl: ', has_jl) print('---') steps = ['run_load_jl', 'run_delete_ori', 'run_entity_nt', 'run_event_nt', 'run_relation_nt', 'run_insert_proto', 'run_super_edge'] print('\n'.join(['Step %d : %s ' % (i, steps[i]) for i in range(len(steps))])) start, end = input('Enter step range you want to run like "0,2"(inclusive)\n').split(',') print('your start end: ', start, end) up = Updater(endpoint, output, graph, True if has_jl == 'True' else False) runs = [up.run_load_jl, up.run_delete_ori, up.run_entity_nt, up.run_event_nt, up.run_relation_nt, up.run_insert_proto, up.run_super_edge] for i in range(int(start), int(end)+1): runs[i]()
format du csv 'id;followers_count;friends_count;favourites_count;statuses_count;lang;text' Ecrit dans un nouveau fichier csv le nombre de retweet ''' if len(argv) < 3: print(argv[0] + ' files.csv updatedCsv') exit(1) csv_files_in = argv[1:-1] csv_file_out = argv[len(argv) - 1] history = argv[len(argv) - 2] for i, csvFileIn in enumerate(csv_files_in): dic = dict() with open(csvFileIn, 'r') as fIn: fIn.readline() for line in fIn: tab = line[:-1].split(';') dic[tab[0]] = tab[1:] updater = Updater(dic) updater.find() for key in updater.dict_of_retweet: dic[key].append(updater.dict_of_retweet[key]) print(i, ' / ', len(csv_files_in)) saveFile(csv_file_out) print("Sleeping for 900 s last fic:" + csvFileIn + ' gathered ', 900 * i, ' / ', 900 * i / len(csv_files_in) * 900 * 100, '%') with open('history', 'a+') as h: h.write(csvFileIn + '\n') time.sleep(900)
def createUpdater(self): if self.opts == None: raise GOFException("Program options have not been initialized") self.updater = Updater(algorithm=self.opts.algorithm)