def start_asynchronous_jobs(self):
"""
Initialize the asynchronous operation, scheduled in the system
"""
if test_reactor:
self._reactor = test_reactor
# Scheduling the Delivery schedule to be executed every 2 seconds
delivery_sched.DeliverySchedule().schedule(2, 1)
# Scheduling the Anomalies Check schedule to be executed every 30 seconds
statistics_sched.AnomaliesSchedule().schedule(30, 2)
# Scheduling the Notification schedule to be executed every 60 seconds
notification_sched.NotificationSchedule().schedule(60, 3)
# Scheduling the Session Management schedule to be executed every minute
session_management_sched.SessionManagementSchedule().schedule(60, 5)
# Scheduling the Tip Cleaning scheduler to be executed every day at 00:00
current_time = datetime_now()
delay = (3600 * (24 + 0)) - (current_time.hour * 3600) - (
current_time.minute * 60) - current_time.second
clean = cleaning_sched.CleaningSchedule().schedule(3600 * 24, delay)
# Scheduling the PGP Check scheduler to be executed every day at 02:00
delay = (3600 * (24 + 2)) - (current_time.hour * 3600) - (
current_time.minute * 60) - current_time.second
pgp_check = pgp_check_sched.PGPCheckSchedule().schedule(
3600 * 24, delay)
# Scheduling the Statistics schedule to be executed every hour on the hour
delay = 3600 - (current_time.minute * 60) - current_time.second
stats = statistics_sched.StatisticsSchedule().schedule(3600, delay)
def test_create_submission_attach_files_finalize_and_verify_file_creation(self):
self.submission_desc = yield self.get_dummy_submission(self.dummyContext['id'])
receipt = yield self.create_submission_with_files(self.submission_desc)
yield delivery_sched.DeliverySchedule().run()
self.fil = yield self.get_internalfiles_by_receipt(receipt)
self.assertTrue(isinstance(self.fil, list))
self.assertEqual(len(self.fil), 3)
self.rfi = yield self.get_receiverfiles_by_receipt(receipt)
self.assertTrue(isinstance(self.rfi, list))
self.assertEqual(len(self.rfi), self.files_created)
counters = {
'encrypted': 0,
'reference': 0
}
for i in range(0, self.files_created):
if self.rfi[i]['status'] not in counters:
counters[self.rfi[i]['status']] = 1
else:
counters[self.rfi[i]['status']] += 1
for key in self.counters_check.keys():
self.assertEqual(counters[key], self.counters_check[key])
def perform_submission(self):
self.dummySubmission['context_id'] = self.dummyContext['id']
self.dummySubmission['receivers'] = self.dummyContext['receivers']
self.dummySubmission['wb_steps'] = yield fill_random_fields(self.dummyContext['id'])
self.dummySubmission = yield create_submission(self.dummySubmission, False, 'en')
yield self.emulate_file_upload(self.dummySubmission['id'])
submission = yield update_submission(self.dummySubmission['id'], self.dummySubmission, True, 'en')
self.dummyWBTip = yield create_whistleblower_tip(self.dummySubmission)
yield delivery_sched.DeliverySchedule().operation()
yield notification_sched.NotificationSchedule().operation()
commentCreation = {
'content': 'comment!',
}
messageCreation = {
'content': 'message!',
}
rtips_desc = yield self.get_rtips()
for rtip_desc in rtips_desc:
yield rtip.create_comment_receiver(rtip_desc['receiver_id'],
rtip_desc['rtip_id'],
commentCreation)
yield rtip.create_message_receiver(rtip_desc['receiver_id'],
rtip_desc['rtip_id'],
messageCreation)
wbtips_desc = yield self.get_wbtips()
for wbtip_desc in wbtips_desc:
yield wbtip.create_comment_wb(wbtip_desc['wbtip_id'],
commentCreation)
for receiver_id in wbtip_desc['wbtip_receivers']:
yield wbtip.create_message_wb(wbtip_desc['wbtip_id'], receiver_id, messageCreation)
yield delivery_sched.DeliverySchedule().operation()
yield notification_sched.NotificationSchedule().operation()
def test_002_tip_life_and_expire(self):
yield self.do_setup_tip_environment()
yield self.do_finalize_submission()
yield delivery_sched.DeliverySchedule().operation()
yield self.check_tip_not_expired()
yield self.force_tip_expire()
yield cleaning_sched.CleaningSchedule().operation()
yield self.test_cleaning()
def start_asynchronous_jobs(self):
"""
Initialize the asynchronous operation, scheduled in the system
"""
if reactor_override:
self._reactor = reactor_override
session_management = session_management_sched.SessionManagementSchedule(
)
self._reactor.callLater(0, session_management.start,
GLSettings.session_management_delta)
anomaly = statistics_sched.AnomaliesSchedule()
self._reactor.callLater(0, anomaly.start, GLSettings.anomaly_delta)
delivery = delivery_sched.DeliverySchedule()
self._reactor.callLater(1, delivery.start, GLSettings.delivery_delta)
notification = notification_sched.NotificationSchedule()
self._reactor.callLater(1, notification.start,
GLSettings.notification_delta)
mailflush = mailflush_sched.MailflushSchedule()
self._reactor.callLater(1, mailflush.start, GLSettings.mailflush_delta)
secure_file_delete = secure_file_delete_sched.SecureFileDeleteSchedule(
)
self._reactor.callLater(1, secure_file_delete.start,
GLSettings.secure_file_delete_delta)
# The Tip cleaning scheduler need to be executed every day at midnight
current_time = datetime_now()
delay = (3600 * 24) - (current_time.hour * 3600) - (
current_time.minute * 60) - current_time.second
clean = cleaning_sched.CleaningSchedule()
self._reactor.callLater(delay, clean.start, 3600 * 24)
# The PGP check scheduler need to be executed every day at midnight
current_time = datetime_now()
delay = (3600 * 24) - (current_time.hour * 3600) - (
current_time.minute * 60) - current_time.second
pgp_check = pgp_check_sched.PGPCheckSchedule()
self._reactor.callLater(delay, pgp_check.start, 3600 * 24)
# The Stats scheduler need to be executed every hour on the hour
current_time = datetime_now()
delay = (60 * 60) - (current_time.minute * 60) - current_time.second
stats = statistics_sched.StatisticsSchedule()
self._reactor.callLater(delay, stats.start, 60 * 60)
def start_asynchronous():
"""
Initialize the asynchronous operation, scheduled in the system
"""
# Scheduled jobs will be started by reactor after reactor.run()
#
# Arguments to reactor.callLater do the following:
# - first argument is the first run delay in seconds
# - second argument is the function that starts the schedule
# - third argument is the schedule period in seconds
session_management = session_management_sched.SessionManagementSchedule()
reactor.callLater(0, session_management.start,
GLSettings.session_management_delta)
anomaly = statistics_sched.AnomaliesSchedule()
reactor.callLater(0, anomaly.start, GLSettings.anomaly_delta)
resources_check = statistics_sched.ResourcesCheckSchedule()
reactor.callLater(0, resources_check.start, GLSettings.anomaly_delta)
delivery = delivery_sched.DeliverySchedule()
reactor.callLater(10, delivery.start, GLSettings.delivery_delta)
notification = notification_sched.NotificationSchedule()
reactor.callLater(20, notification.start, GLSettings.notification_delta)
mailflush = mailflush_sched.MailflushSchedule()
reactor.callLater(40, mailflush.start, GLSettings.mailflush_delta)
# The Tip cleaning scheduler need to be executed every day at midnight
current_time = datetime_now()
delay = (3600 * 24) - (current_time.hour * 3600) - (
current_time.minute * 60) - current_time.second
clean = cleaning_sched.CleaningSchedule()
reactor.callLater(delay, clean.start, 3600 * 24)
# The PGP check scheduler need to be executed every day at midnight
current_time = datetime_now()
delay = (3600 * 24) - (current_time.hour * 3600) - (
current_time.minute * 60) - current_time.second
pgp_check = pgp_check_sched.PGPCheckSchedule()
reactor.callLater(delay, pgp_check.start, 3600 * 24)
# The Stats scheduler need to be executed every hour on the hour
current_time = datetime_now()
delay = (60 * 60) - (current_time.minute * 60) - current_time.second
stats = statistics_sched.StatisticsSchedule()
reactor.callLater(delay, stats.start, 60 * 60)
def test_004_tip_life_and_expire_with_files(self):
yield self.do_setup_tip_environment()
yield self.emulate_file_upload(self.submission_desc['id'])
yield self.do_finalize_submission()
yield delivery_sched.DeliverySchedule().operation()
self.assertTrue(os.listdir(GLSetting.submission_path) != [])
yield self.check_tip_not_expired()
yield self.force_tip_expire()
yield cleaning_sched.CleaningSchedule().operation()
self.assertTrue(os.listdir(GLSetting.submission_path) == [])
self.assertTrue(os.listdir(GLSetting.tmp_upload_path) == [])
def start_asynchronous_jobs(self):
"""
Initialize the asynchronous operation, scheduled in the system
"""
if test_reactor:
self._reactor = test_reactor
# Scheduling the Delivery schedule to be executed every 2 seconds
delivery = delivery_sched.DeliverySchedule()
self._reactor.callLater(1, delivery.start, 2)
# Scheduling the Anomalies Check schedule to be executed every 30 seconds
anomaly = statistics_sched.AnomaliesSchedule()
self._reactor.callLater(0, anomaly.start, 30)
# Scheduling the Notification schedule to be executed every 60 seconds
notification = notification_sched.NotificationSchedule()
self._reactor.callLater(1, notification.start, 60)
# Scheduling the Session Management schedule to be executed every minute
session_management = session_management_sched.SessionManagementSchedule(
)
self._reactor.callLater(0, session_management.start, 60)
# Scheduling the Tip Cleaning scheduler to be executed every day at 00:00
current_time = datetime_now()
delay = (3600 * 24) - (current_time.hour * 3600) - (
current_time.minute * 60) - current_time.second
clean = cleaning_sched.CleaningSchedule()
self._reactor.callLater(delay, clean.start, 3600 * 24)
# Scheduling the PGP Check scheduler to be executed every day at 01:00
current_time = datetime_now()
delay = (3600 * 25) - (current_time.hour * 3600) - (
current_time.minute * 60) - current_time.second
pgp_check = pgp_check_sched.PGPCheckSchedule()
self._reactor.callLater(delay, pgp_check.start, 3600 * 24)
# Scheduling the Statistics schedule to be executed every hour on the hour
current_time = datetime_now()
delay = (60 * 60) - (current_time.minute * 60) - current_time.second
stats = statistics_sched.StatisticsSchedule()
self._reactor.callLater(delay, stats.start, 60 * 60)
def test_create_receiverfiles_allow_unencrypted_true_no_keys_loaded(self):
yield self.test_create_submission_attach_files_finalize_and_access_wbtip(
)
yield delivery_sched.DeliverySchedule().operation()
self.fil = yield self.get_internalfiles_by_wbtip(
self.submission_desc['id'])
self.assertTrue(isinstance(self.fil, list))
self.assertEqual(len(self.fil), 3)
self.rfi = yield self.get_receiverfiles_by_wbtip(
self.submission_desc['id'])
self.assertTrue(isinstance(self.rfi, list))
self.assertEqual(len(self.rfi), 6)
for i in range(0, 6):
self.assertTrue(
self.rfi[i]['status'] in [u'reference', u'encrypted'])
def start_asynchronous():
"""
Initialize the asynchronous operation, scheduled in the system
"""
# Here we prepare the scheduled,
# schedules will be started by reactor after reactor.run()
session_management = session_management_sched.SessionManagementSchedule()
delivery = delivery_sched.DeliverySchedule()
notification = notification_sched.NotificationSchedule()
clean = cleaning_sched.CleaningSchedule()
pgp_check = pgp_check_sched.PGPCheckSchedule()
mailflush = mailflush_sched.MailflushSchedule()
resource_check = statistics_sched.ResourceChecker()
anomaly = statistics_sched.AnomaliesSchedule()
stats = statistics_sched.StatisticsSchedule()
# here we prepare the schedule:
# - first argument is the first run delay in seconds
# - second argument is the function that starts the schedule
# - third argument is the schedule period in seconds
reactor.callLater(0, session_management.start, GLSetting.session_management_minutes_delta * 60)
reactor.callLater(10, delivery.start, GLSetting.delivery_seconds_delta)
reactor.callLater(20, notification.start, GLSetting.notification_minutes_delta * 60)
reactor.callLater(30, clean.start, GLSetting.cleaning_hours_delta * 3600)
reactor.callLater(40, mailflush.start, GLSetting.mailflush_minutes_delta * 60)
reactor.callLater(50, resource_check.start, GLSetting.anomaly_seconds_delta)
reactor.callLater(60, anomaly.start, GLSetting.anomaly_seconds_delta)
# The Stats scheduler need to be executed every hour on the hour.
current_time = datetime_now()
delay = (60 * 60) - (current_time.minute * 60) - current_time.second
reactor.callLater(delay, stats.start, 60 * 60)
statistics_sched.StatisticsSchedule.collection_start_datetime = current_time
# The PGP check scheduler need to be executed every day at midnight
current_time = datetime_now()
delay = (3600 * 24) - (current_time.hour * 3600) - (current_time.minute * 60) - current_time.second
reactor.callLater(delay, pgp_check.start, 3600 * 24)
def start_asynchronous():
"""
Initialize the asynchronous operation, scheduled in the system
https://github.com/globaleaks/GLBackend/wiki/Asynchronous-and-synchronous-operation
This method would be likely put in GLBaseRunner.postApplication, but is
not executed by globaleaks.run_app, then is called by the
OS-depenedent runner below
"""
from globaleaks.jobs import session_management_sched, statistics_sched, \
notification_sched, delivery_sched, cleaning_sched, \
pgp_check_sched
# Here we prepare the scheduled, schedules will be started by reactor after reactor.run()
session_management = session_management_sched.SessionManagementSchedule()
delivery = delivery_sched.DeliverySchedule()
notification = notification_sched.NotificationSchedule()
clean = cleaning_sched.CleaningSchedule()
anomaly = statistics_sched.AnomaliesSchedule()
stats = statistics_sched.StatisticsSchedule()
pgp_check = pgp_check_sched.PGPCheckSchedule()
# here we prepare the schedule:
# - first argument is the first run delay in seconds
# - second argument is the function that starts the schedule
# - third argument is the schedule period in seconds
reactor.callLater(0, session_management.start,
GLSetting.session_management_minutes_delta * 60)
reactor.callLater(10, delivery.start, GLSetting.delivery_seconds_delta)
reactor.callLater(20, notification.start,
GLSetting.notification_minutes_delta * 60)
reactor.callLater(30, clean.start, GLSetting.cleaning_hours_delta * 3600)
reactor.callLater(40, anomaly.start, GLSetting.anomaly_seconds_delta)
reactor.callLater(50, stats.start, GLSetting.stats_minutes_delta * 60)
reactor.callLater(60, pgp_check.start,
GLSetting.pgp_check_hours_delta * 3600)
def fill_data(self):
try:
yield do_appdata_init()
except Exception as excp:
print "Fail fill_data/do_appdata_init: %s" % excp
raise excp
receivers_ids = []
try:
self.dummyReceiver_1 = yield create_receiver(self.dummyReceiver_1)
receivers_ids.append(self.dummyReceiver_1['id'])
self.dummyReceiver_2 = yield create_receiver(self.dummyReceiver_2)
receivers_ids.append(self.dummyReceiver_2['id'])
except Exception as excp:
print "Fail fill_data/create_receiver: %s" % excp
raise excp
try:
self.dummyContext['receivers'] = receivers_ids
self.dummyContext = yield create_context(self.dummyContext)
except Exception as excp:
print "Fail fill_data/create_context: %s" % excp
raise excp
self.dummySubmission['context_id'] = self.dummyContext['id']
self.dummySubmission['receivers'] = receivers_ids
self.dummySubmission['wb_fields'] = fill_random_fields(
self.dummyContext)
try:
self.dummySubmissionNotFinalized = yield create_submission(
self.dummySubmission, finalize=False)
except Exception as excp:
print "Fail fill_data/create_submission: %s" % excp
raise excp
try:
self.dummySubmission = yield create_submission(
self.dummySubmission, finalize=False)
except Exception as excp:
print "Fail fill_data/create_submission: %s" % excp
raise excp
yield self.emulate_file_upload(self.dummySubmission['id'])
try:
submission = yield update_submission(self.dummySubmission['id'],
self.dummySubmission,
finalize=True)
except Exception as excp:
print "Fail fill_data/update_submission: %s" % excp
raise excp
try:
self.dummyWBTip = yield create_whistleblower_tip(
self.dummySubmission)
except Exception as excp:
print "Fail fill_data/create_whistleblower: %s" % excp
raise excp
assert self.dummyReceiver_1.has_key('id')
assert self.dummyReceiver_2.has_key('id')
assert self.dummyContext.has_key('id')
assert self.dummySubmission.has_key('id')
yield delivery_sched.DeliverySchedule().operation()
yield notification_sched.NotificationSchedule().operation()
commentCreation = {
'content': 'comment!',
}
messageCreation = {
'content': 'message!',
}
rtips_desc = yield self.get_rtips()
for rtip_desc in rtips_desc:
yield rtip.create_comment_receiver(rtip_desc['receiver_id'],
rtip_desc['rtip_id'],
commentCreation)
yield rtip.create_message_receiver(rtip_desc['receiver_id'],
rtip_desc['rtip_id'],
messageCreation)
wbtips_desc = yield self.get_wbtips()
for wbtip_desc in wbtips_desc:
yield wbtip.create_comment_wb(wbtip_desc['wbtip_id'],
commentCreation)
for receiver_id in wbtip_desc['wbtip_receivers']:
yield wbtip.create_message_wb(wbtip_desc['wbtip_id'],
receiver_id, messageCreation)
yield delivery_sched.DeliverySchedule().operation()
yield notification_sched.NotificationSchedule().operation()