def start_asynchronous_jobs(self): """ Initialize the asynchronous operation, scheduled in the system """ if test_reactor: self._reactor = test_reactor # Scheduling the Delivery schedule to be executed every 2 seconds delivery_sched.DeliverySchedule().schedule(2, 1) # Scheduling the Anomalies Check schedule to be executed every 30 seconds statistics_sched.AnomaliesSchedule().schedule(30, 2) # Scheduling the Notification schedule to be executed every 60 seconds notification_sched.NotificationSchedule().schedule(60, 3) # Scheduling the Session Management schedule to be executed every minute session_management_sched.SessionManagementSchedule().schedule(60, 5) # Scheduling the Tip Cleaning scheduler to be executed every day at 00:00 current_time = datetime_now() delay = (3600 * (24 + 0)) - (current_time.hour * 3600) - ( current_time.minute * 60) - current_time.second clean = cleaning_sched.CleaningSchedule().schedule(3600 * 24, delay) # Scheduling the PGP Check scheduler to be executed every day at 02:00 delay = (3600 * (24 + 2)) - (current_time.hour * 3600) - ( current_time.minute * 60) - current_time.second pgp_check = pgp_check_sched.PGPCheckSchedule().schedule( 3600 * 24, delay) # Scheduling the Statistics schedule to be executed every hour on the hour delay = 3600 - (current_time.minute * 60) - current_time.second stats = statistics_sched.StatisticsSchedule().schedule(3600, delay)
def test_create_submission_attach_files_finalize_and_verify_file_creation(self): self.submission_desc = yield self.get_dummy_submission(self.dummyContext['id']) receipt = yield self.create_submission_with_files(self.submission_desc) yield delivery_sched.DeliverySchedule().run() self.fil = yield self.get_internalfiles_by_receipt(receipt) self.assertTrue(isinstance(self.fil, list)) self.assertEqual(len(self.fil), 3) self.rfi = yield self.get_receiverfiles_by_receipt(receipt) self.assertTrue(isinstance(self.rfi, list)) self.assertEqual(len(self.rfi), self.files_created) counters = { 'encrypted': 0, 'reference': 0 } for i in range(0, self.files_created): if self.rfi[i]['status'] not in counters: counters[self.rfi[i]['status']] = 1 else: counters[self.rfi[i]['status']] += 1 for key in self.counters_check.keys(): self.assertEqual(counters[key], self.counters_check[key])
def perform_submission(self): self.dummySubmission['context_id'] = self.dummyContext['id'] self.dummySubmission['receivers'] = self.dummyContext['receivers'] self.dummySubmission['wb_steps'] = yield fill_random_fields(self.dummyContext['id']) self.dummySubmission = yield create_submission(self.dummySubmission, False, 'en') yield self.emulate_file_upload(self.dummySubmission['id']) submission = yield update_submission(self.dummySubmission['id'], self.dummySubmission, True, 'en') self.dummyWBTip = yield create_whistleblower_tip(self.dummySubmission) yield delivery_sched.DeliverySchedule().operation() yield notification_sched.NotificationSchedule().operation() commentCreation = { 'content': 'comment!', } messageCreation = { 'content': 'message!', } rtips_desc = yield self.get_rtips() for rtip_desc in rtips_desc: yield rtip.create_comment_receiver(rtip_desc['receiver_id'], rtip_desc['rtip_id'], commentCreation) yield rtip.create_message_receiver(rtip_desc['receiver_id'], rtip_desc['rtip_id'], messageCreation) wbtips_desc = yield self.get_wbtips() for wbtip_desc in wbtips_desc: yield wbtip.create_comment_wb(wbtip_desc['wbtip_id'], commentCreation) for receiver_id in wbtip_desc['wbtip_receivers']: yield wbtip.create_message_wb(wbtip_desc['wbtip_id'], receiver_id, messageCreation) yield delivery_sched.DeliverySchedule().operation() yield notification_sched.NotificationSchedule().operation()
def test_002_tip_life_and_expire(self): yield self.do_setup_tip_environment() yield self.do_finalize_submission() yield delivery_sched.DeliverySchedule().operation() yield self.check_tip_not_expired() yield self.force_tip_expire() yield cleaning_sched.CleaningSchedule().operation() yield self.test_cleaning()
def start_asynchronous_jobs(self): """ Initialize the asynchronous operation, scheduled in the system """ if reactor_override: self._reactor = reactor_override session_management = session_management_sched.SessionManagementSchedule( ) self._reactor.callLater(0, session_management.start, GLSettings.session_management_delta) anomaly = statistics_sched.AnomaliesSchedule() self._reactor.callLater(0, anomaly.start, GLSettings.anomaly_delta) delivery = delivery_sched.DeliverySchedule() self._reactor.callLater(1, delivery.start, GLSettings.delivery_delta) notification = notification_sched.NotificationSchedule() self._reactor.callLater(1, notification.start, GLSettings.notification_delta) mailflush = mailflush_sched.MailflushSchedule() self._reactor.callLater(1, mailflush.start, GLSettings.mailflush_delta) secure_file_delete = secure_file_delete_sched.SecureFileDeleteSchedule( ) self._reactor.callLater(1, secure_file_delete.start, GLSettings.secure_file_delete_delta) # The Tip cleaning scheduler need to be executed every day at midnight current_time = datetime_now() delay = (3600 * 24) - (current_time.hour * 3600) - ( current_time.minute * 60) - current_time.second clean = cleaning_sched.CleaningSchedule() self._reactor.callLater(delay, clean.start, 3600 * 24) # The PGP check scheduler need to be executed every day at midnight current_time = datetime_now() delay = (3600 * 24) - (current_time.hour * 3600) - ( current_time.minute * 60) - current_time.second pgp_check = pgp_check_sched.PGPCheckSchedule() self._reactor.callLater(delay, pgp_check.start, 3600 * 24) # The Stats scheduler need to be executed every hour on the hour current_time = datetime_now() delay = (60 * 60) - (current_time.minute * 60) - current_time.second stats = statistics_sched.StatisticsSchedule() self._reactor.callLater(delay, stats.start, 60 * 60)
def start_asynchronous(): """ Initialize the asynchronous operation, scheduled in the system """ # Scheduled jobs will be started by reactor after reactor.run() # # Arguments to reactor.callLater do the following: # - first argument is the first run delay in seconds # - second argument is the function that starts the schedule # - third argument is the schedule period in seconds session_management = session_management_sched.SessionManagementSchedule() reactor.callLater(0, session_management.start, GLSettings.session_management_delta) anomaly = statistics_sched.AnomaliesSchedule() reactor.callLater(0, anomaly.start, GLSettings.anomaly_delta) resources_check = statistics_sched.ResourcesCheckSchedule() reactor.callLater(0, resources_check.start, GLSettings.anomaly_delta) delivery = delivery_sched.DeliverySchedule() reactor.callLater(10, delivery.start, GLSettings.delivery_delta) notification = notification_sched.NotificationSchedule() reactor.callLater(20, notification.start, GLSettings.notification_delta) mailflush = mailflush_sched.MailflushSchedule() reactor.callLater(40, mailflush.start, GLSettings.mailflush_delta) # The Tip cleaning scheduler need to be executed every day at midnight current_time = datetime_now() delay = (3600 * 24) - (current_time.hour * 3600) - ( current_time.minute * 60) - current_time.second clean = cleaning_sched.CleaningSchedule() reactor.callLater(delay, clean.start, 3600 * 24) # The PGP check scheduler need to be executed every day at midnight current_time = datetime_now() delay = (3600 * 24) - (current_time.hour * 3600) - ( current_time.minute * 60) - current_time.second pgp_check = pgp_check_sched.PGPCheckSchedule() reactor.callLater(delay, pgp_check.start, 3600 * 24) # The Stats scheduler need to be executed every hour on the hour current_time = datetime_now() delay = (60 * 60) - (current_time.minute * 60) - current_time.second stats = statistics_sched.StatisticsSchedule() reactor.callLater(delay, stats.start, 60 * 60)
def test_004_tip_life_and_expire_with_files(self): yield self.do_setup_tip_environment() yield self.emulate_file_upload(self.submission_desc['id']) yield self.do_finalize_submission() yield delivery_sched.DeliverySchedule().operation() self.assertTrue(os.listdir(GLSetting.submission_path) != []) yield self.check_tip_not_expired() yield self.force_tip_expire() yield cleaning_sched.CleaningSchedule().operation() self.assertTrue(os.listdir(GLSetting.submission_path) == []) self.assertTrue(os.listdir(GLSetting.tmp_upload_path) == [])
def start_asynchronous_jobs(self): """ Initialize the asynchronous operation, scheduled in the system """ if test_reactor: self._reactor = test_reactor # Scheduling the Delivery schedule to be executed every 2 seconds delivery = delivery_sched.DeliverySchedule() self._reactor.callLater(1, delivery.start, 2) # Scheduling the Anomalies Check schedule to be executed every 30 seconds anomaly = statistics_sched.AnomaliesSchedule() self._reactor.callLater(0, anomaly.start, 30) # Scheduling the Notification schedule to be executed every 60 seconds notification = notification_sched.NotificationSchedule() self._reactor.callLater(1, notification.start, 60) # Scheduling the Session Management schedule to be executed every minute session_management = session_management_sched.SessionManagementSchedule( ) self._reactor.callLater(0, session_management.start, 60) # Scheduling the Tip Cleaning scheduler to be executed every day at 00:00 current_time = datetime_now() delay = (3600 * 24) - (current_time.hour * 3600) - ( current_time.minute * 60) - current_time.second clean = cleaning_sched.CleaningSchedule() self._reactor.callLater(delay, clean.start, 3600 * 24) # Scheduling the PGP Check scheduler to be executed every day at 01:00 current_time = datetime_now() delay = (3600 * 25) - (current_time.hour * 3600) - ( current_time.minute * 60) - current_time.second pgp_check = pgp_check_sched.PGPCheckSchedule() self._reactor.callLater(delay, pgp_check.start, 3600 * 24) # Scheduling the Statistics schedule to be executed every hour on the hour current_time = datetime_now() delay = (60 * 60) - (current_time.minute * 60) - current_time.second stats = statistics_sched.StatisticsSchedule() self._reactor.callLater(delay, stats.start, 60 * 60)
def test_create_receiverfiles_allow_unencrypted_true_no_keys_loaded(self): yield self.test_create_submission_attach_files_finalize_and_access_wbtip( ) yield delivery_sched.DeliverySchedule().operation() self.fil = yield self.get_internalfiles_by_wbtip( self.submission_desc['id']) self.assertTrue(isinstance(self.fil, list)) self.assertEqual(len(self.fil), 3) self.rfi = yield self.get_receiverfiles_by_wbtip( self.submission_desc['id']) self.assertTrue(isinstance(self.rfi, list)) self.assertEqual(len(self.rfi), 6) for i in range(0, 6): self.assertTrue( self.rfi[i]['status'] in [u'reference', u'encrypted'])
def start_asynchronous(): """ Initialize the asynchronous operation, scheduled in the system """ # Here we prepare the scheduled, # schedules will be started by reactor after reactor.run() session_management = session_management_sched.SessionManagementSchedule() delivery = delivery_sched.DeliverySchedule() notification = notification_sched.NotificationSchedule() clean = cleaning_sched.CleaningSchedule() pgp_check = pgp_check_sched.PGPCheckSchedule() mailflush = mailflush_sched.MailflushSchedule() resource_check = statistics_sched.ResourceChecker() anomaly = statistics_sched.AnomaliesSchedule() stats = statistics_sched.StatisticsSchedule() # here we prepare the schedule: # - first argument is the first run delay in seconds # - second argument is the function that starts the schedule # - third argument is the schedule period in seconds reactor.callLater(0, session_management.start, GLSetting.session_management_minutes_delta * 60) reactor.callLater(10, delivery.start, GLSetting.delivery_seconds_delta) reactor.callLater(20, notification.start, GLSetting.notification_minutes_delta * 60) reactor.callLater(30, clean.start, GLSetting.cleaning_hours_delta * 3600) reactor.callLater(40, mailflush.start, GLSetting.mailflush_minutes_delta * 60) reactor.callLater(50, resource_check.start, GLSetting.anomaly_seconds_delta) reactor.callLater(60, anomaly.start, GLSetting.anomaly_seconds_delta) # The Stats scheduler need to be executed every hour on the hour. current_time = datetime_now() delay = (60 * 60) - (current_time.minute * 60) - current_time.second reactor.callLater(delay, stats.start, 60 * 60) statistics_sched.StatisticsSchedule.collection_start_datetime = current_time # The PGP check scheduler need to be executed every day at midnight current_time = datetime_now() delay = (3600 * 24) - (current_time.hour * 3600) - (current_time.minute * 60) - current_time.second reactor.callLater(delay, pgp_check.start, 3600 * 24)
def start_asynchronous(): """ Initialize the asynchronous operation, scheduled in the system https://github.com/globaleaks/GLBackend/wiki/Asynchronous-and-synchronous-operation This method would be likely put in GLBaseRunner.postApplication, but is not executed by globaleaks.run_app, then is called by the OS-depenedent runner below """ from globaleaks.jobs import session_management_sched, statistics_sched, \ notification_sched, delivery_sched, cleaning_sched, \ pgp_check_sched # Here we prepare the scheduled, schedules will be started by reactor after reactor.run() session_management = session_management_sched.SessionManagementSchedule() delivery = delivery_sched.DeliverySchedule() notification = notification_sched.NotificationSchedule() clean = cleaning_sched.CleaningSchedule() anomaly = statistics_sched.AnomaliesSchedule() stats = statistics_sched.StatisticsSchedule() pgp_check = pgp_check_sched.PGPCheckSchedule() # here we prepare the schedule: # - first argument is the first run delay in seconds # - second argument is the function that starts the schedule # - third argument is the schedule period in seconds reactor.callLater(0, session_management.start, GLSetting.session_management_minutes_delta * 60) reactor.callLater(10, delivery.start, GLSetting.delivery_seconds_delta) reactor.callLater(20, notification.start, GLSetting.notification_minutes_delta * 60) reactor.callLater(30, clean.start, GLSetting.cleaning_hours_delta * 3600) reactor.callLater(40, anomaly.start, GLSetting.anomaly_seconds_delta) reactor.callLater(50, stats.start, GLSetting.stats_minutes_delta * 60) reactor.callLater(60, pgp_check.start, GLSetting.pgp_check_hours_delta * 3600)
def fill_data(self): try: yield do_appdata_init() except Exception as excp: print "Fail fill_data/do_appdata_init: %s" % excp raise excp receivers_ids = [] try: self.dummyReceiver_1 = yield create_receiver(self.dummyReceiver_1) receivers_ids.append(self.dummyReceiver_1['id']) self.dummyReceiver_2 = yield create_receiver(self.dummyReceiver_2) receivers_ids.append(self.dummyReceiver_2['id']) except Exception as excp: print "Fail fill_data/create_receiver: %s" % excp raise excp try: self.dummyContext['receivers'] = receivers_ids self.dummyContext = yield create_context(self.dummyContext) except Exception as excp: print "Fail fill_data/create_context: %s" % excp raise excp self.dummySubmission['context_id'] = self.dummyContext['id'] self.dummySubmission['receivers'] = receivers_ids self.dummySubmission['wb_fields'] = fill_random_fields( self.dummyContext) try: self.dummySubmissionNotFinalized = yield create_submission( self.dummySubmission, finalize=False) except Exception as excp: print "Fail fill_data/create_submission: %s" % excp raise excp try: self.dummySubmission = yield create_submission( self.dummySubmission, finalize=False) except Exception as excp: print "Fail fill_data/create_submission: %s" % excp raise excp yield self.emulate_file_upload(self.dummySubmission['id']) try: submission = yield update_submission(self.dummySubmission['id'], self.dummySubmission, finalize=True) except Exception as excp: print "Fail fill_data/update_submission: %s" % excp raise excp try: self.dummyWBTip = yield create_whistleblower_tip( self.dummySubmission) except Exception as excp: print "Fail fill_data/create_whistleblower: %s" % excp raise excp assert self.dummyReceiver_1.has_key('id') assert self.dummyReceiver_2.has_key('id') assert self.dummyContext.has_key('id') assert self.dummySubmission.has_key('id') yield delivery_sched.DeliverySchedule().operation() yield notification_sched.NotificationSchedule().operation() commentCreation = { 'content': 'comment!', } messageCreation = { 'content': 'message!', } rtips_desc = yield self.get_rtips() for rtip_desc in rtips_desc: yield rtip.create_comment_receiver(rtip_desc['receiver_id'], rtip_desc['rtip_id'], commentCreation) yield rtip.create_message_receiver(rtip_desc['receiver_id'], rtip_desc['rtip_id'], messageCreation) wbtips_desc = yield self.get_wbtips() for wbtip_desc in wbtips_desc: yield wbtip.create_comment_wb(wbtip_desc['wbtip_id'], commentCreation) for receiver_id in wbtip_desc['wbtip_receivers']: yield wbtip.create_message_wb(wbtip_desc['wbtip_id'], receiver_id, messageCreation) yield delivery_sched.DeliverySchedule().operation() yield notification_sched.NotificationSchedule().operation()