def run_manual_code(study_id):
"""
Create an AWS Batch job for the Study specified
:param study_id: Primary key of a Study
"""
# we assume that the cluster is configured only in one region.
pipeline_region = get_current_region()
# Get the object ID of the study, used in the pipeline
query = Study.objects.filter(pk=study_id)
if not query.exists():
return abort(404)
object_id = query.get().object_id
error_sentry = make_error_sentry("data",
tags={"pipeline_frequency": "manually"})
# Get new data access credentials for the manual user, submit a manual job, display message
# Report all errors to sentry including DataPipelineNotConfigured errors.
with error_sentry:
ssm_client = get_boto_client('ssm', pipeline_region)
refresh_data_access_credentials('manually',
ssm_client=ssm_client,
webserver=True)
batch_client = get_boto_client('batch', pipeline_region)
create_one_job('manually', object_id, batch_client, webserver=True)
flash('Data pipeline code successfully initiated!', 'success')
if error_sentry.errors:
flash('An unknown error occurred when trying to run this task.',
category='danger')
print error_sentry
return redirect('/data-pipeline/{:s}'.format(study_id))
def get_boto_client(client_type, pipeline_region=None):
from config.settings import BEIWE_SERVER_AWS_ACCESS_KEY_ID, BEIWE_SERVER_AWS_SECRET_ACCESS_KEY
region_name = pipeline_region or get_current_region()
return boto3.client(
client_type,
aws_access_key_id=BEIWE_SERVER_AWS_ACCESS_KEY_ID,
aws_secret_access_key=BEIWE_SERVER_AWS_SECRET_ACCESS_KEY,
region_name=region_name,
)
_one_folder_up = _abspath(__file__).rsplit('/', 2)[0]
_path.insert(1, _one_folder_up)
from datetime import timedelta
from django.utils import timezone
from database.data_access_models import ChunkRegistry
from database.study_models import Study
from libs.sentry import make_error_sentry
from pipeline.boto_helpers import get_boto_client
from pipeline.configuration_getters import get_current_region
from pipeline.index import create_one_job, refresh_data_access_credentials
pipeline_region = get_current_region()
ssm_client = get_boto_client('ssm', pipeline_region)
error_sentry = make_error_sentry("data",
tags={"pipeline_frequency": "manually"})
batch_client = get_boto_client('batch', pipeline_region)
yesterday = timezone.now() - timedelta(days=1)
refresh_data_access_credentials('manually',
ssm_client=ssm_client,
webserver=False)
################################################################################################
# if you are running this on an ubuntu machine you have to sudo apt-get -y install cloud-utils #
################################################################################################
for study in Study.objects.all():