def test_setup_settings(self):
config = {'sftp.jail.base_path': '/jail',
'extract.sftp.route.base_dir': 'newRoute'}
setup_settings(config)
self.assertEqual(get_setting(Config.MAX_RETRIES), 1)
self.assertEqual(get_setting(Config.TIMEOUT), 20)
self.assertEqual(get_setting(Config.PICKUP_ROUTE_BASE_DIR), 'newRoute')
def archive_with_encryption(request_id, recipients, archive_file_name,
directory):
'''
given a directory, archive everything in this directory to a file name specified
'''
retryable = False
exception_thrown = False
try:
task_info = {
Constants.TASK_ID: archive_with_encryption.request.id,
Constants.CELERY_TASK_ID: archive_with_encryption.request.id,
Constants.REQUEST_GUID: request_id
}
insert_extract_stats(task_info,
{Constants.STATUS: ExtractStatus.ARCHIVING})
gpg_binary_file = get_setting(Config.BINARYFILE)
homedir = get_setting(Config.HOMEDIR)
keyserver = get_setting(Config.KEYSERVER)
encrypted_archive_files(directory,
recipients,
archive_file_name,
homedir=homedir,
keyserver=keyserver,
gpgbinary=gpg_binary_file)
insert_extract_stats(task_info,
{Constants.STATUS: ExtractStatus.ARCHIVED})
except GPGPublicKeyException as e:
# recoverable exception
retryable = True
exception_thrown = True
insert_extract_stats(task_info, {
Constants.STATUS: ExtractStatus.FAILED,
Constants.INFO: str(e)
})
except Exception as e:
# unrecoverable exception
exception_thrown = True
insert_extract_stats(task_info, {
Constants.STATUS: ExtractStatus.FAILED,
Constants.INFO: str(e)
})
if exception_thrown:
if retryable:
try:
# this looks funny to you, but this is just a working around solution for celery bug
# since exc option is not really working for retry.
raise ExtractionError()
except ExtractionError as exc:
raise archive_with_encryption.retry(args=[
request_id, recipients, archive_file_name, directory
],
exc=exc)
else:
raise ExtractionError()
def test_setup_settings(self):
config = {
'sftp.jail.base_path': '/jail',
'extract.sftp.route.base_dir': 'newRoute'
}
setup_settings(config)
self.assertEqual(get_setting(Config.MAX_RETRIES), 1)
self.assertEqual(get_setting(Config.TIMEOUT), 20)
self.assertEqual(get_setting(Config.PICKUP_ROUTE_BASE_DIR), 'newRoute')
def archive_with_encryption(request_id, recipients, archive_file_name, directory):
'''
given a directory, archive everything in this directory to a file name specified
'''
retryable = False
exception_thrown = False
try:
task_info = {Constants.TASK_ID: archive_with_encryption.request.id,
Constants.CELERY_TASK_ID: archive_with_encryption.request.id,
Constants.REQUEST_GUID: request_id}
insert_extract_stats(task_info, {Constants.STATUS: ExtractStatus.ARCHIVING})
gpg_binary_file = get_setting(Config.BINARYFILE)
homedir = get_setting(Config.HOMEDIR)
keyserver = get_setting(Config.KEYSERVER)
encrypted_archive_files(directory, recipients, archive_file_name, homedir=homedir, keyserver=keyserver, gpgbinary=gpg_binary_file)
insert_extract_stats(task_info, {Constants.STATUS: ExtractStatus.ARCHIVED})
except GPGPublicKeyException as e:
# recoverable exception
retryable = True
exception_thrown = True
insert_extract_stats(task_info, {Constants.STATUS: ExtractStatus.FAILED, Constants.INFO: str(e)})
except Exception as e:
# unrecoverable exception
exception_thrown = True
insert_extract_stats(task_info, {Constants.STATUS: ExtractStatus.FAILED, Constants.INFO: str(e)})
if exception_thrown:
if retryable:
try:
# this looks funny to you, but this is just a working around solution for celery bug
# since exc option is not really working for retry.
raise ExtractionError()
except ExtractionError as exc:
raise archive_with_encryption.retry(args=[request_id, recipients, archive_file_name, directory], exc=exc)
else:
raise ExtractionError()
def test_get_setting(self):
self.assertIsNone(get_setting("nothing"))
def test_defaults_in_settings(self):
config = {}
setup_settings(config)
self.assertEqual(get_setting(Config.PICKUP_ROUTE_BASE_DIR), 'route')
# of such consortium member educational agencies.
__author__ = 'tshewchuk'
"""
This module contains the previous archive task, renamed to archive_with_encryption.
It's a placeholder for the no longer used archive task which included encryption.
"""
from edextract.settings.config import Config, get_setting
from edextract.celery import celery
from edextract.status.constants import Constants
from edextract.status.status import ExtractStatus, insert_extract_stats
from edcore.utils.data_archiver import encrypted_archive_files, GPGPublicKeyException
from edextract.exceptions import ExtractionError
MAX_RETRY = get_setting(Config.MAX_RETRIES, 1)
DEFAULT_RETRY_DELAY = get_setting(Config.RETRY_DELAY, 60)
@celery.task(name="tasks.archive_with_encryption.archive_with_encryption",
max_retries=MAX_RETRY,
default_retry_delay=DEFAULT_RETRY_DELAY)
def archive_with_encryption(request_id, recipients, archive_file_name,
directory):
'''
given a directory, archive everything in this directory to a file name specified
'''
retryable = False
exception_thrown = False
def test_get_setting(self):
self.assertIsNone(get_setting("nothing"))
def test_defaults_in_settings(self):
config = {}
setup_settings(config)
self.assertEqual(get_setting(Config.PICKUP_ROUTE_BASE_DIR), 'route')
def copy(filename, hostname, tenant, gatekeeper, sftp_username, private_key_file, timeout=1800):
SendFileUtil.remote_transfer_file(source_file=filename, hostname=hostname,
remote_base_dir=get_setting(Config.PICKUP_ROUTE_BASE_DIR),
file_tenantname=tenant, file_username=gatekeeper,
sftp_username=sftp_username, private_key_file=private_key_file,
timeout=timeout)
from edextract.exceptions import ExtractionError
from edcore.exceptions import RemoteCopyError
from edcore.utils.utils import archive_files
from edextract.data_extract_generation.query_extract_generator import generate_csv, generate_json
from edextract.data_extract_generation.item_level_generator import generate_items_csv
from edextract.data_extract_generation.raw_data_generator import generate_raw_data_xml
from edextract.data_extract_generation.student_reg_report_generator import generate_statistics_report, generate_completion_report
from edextract.tasks.constants import ExtractionDataType
from email.mime.text import MIMEText
from hpz_client.frs.http_file_upload import http_file_upload
import shutil
import smtplib
log = logging.getLogger('edextract')
MAX_RETRY = get_setting(Config.MAX_RETRIES, 1)
DEFAULT_RETRY_DELAY = get_setting(Config.RETRY_DELAY, 60)
@celery.task(name="tasks.extract.separator")
def extract_group_separator():
'''
A dummy task to separate out a chain of two consecutive groups
'''
pass
@celery.task(name='task.extract.prepare_path')
def prepare_path(request_id, paths):
'''
Given a list of paths of directories, creates it if it doesn't exist