def clean_experiment(accession, email=None):
# conan = ConanPage(url=settings.CONAN_URL)
if not email:
email = settings.CONAN_LOGIN_EMAIL
# conan.login(login_email=email)
# conan.clean_experiment(accession)
submit_conan_task(accession=accession,
pipeline_name=CONAN_PIPELINES.unload,
starting_index=1)
job_status = retrieve_job_status(accession)
while job_status != 'COMPLETED':
if job_status == 'FAILED':
raise Exception('Clean Failed for ' + accession)
time.sleep(30)
job_status = retrieve_job_status(accession)
def unload_experiment(accession):
# conan = ConanPage(url=settings.CONAN_URL)
# conan.login(login_email=settings.CONAN_LOGIN_EMAIL)
print 'unloading : ', accession
# conan.unload_experiment(accession)
submit_conan_task(accession=accession,
pipeline_name=CONAN_PIPELINES.unload)
job_status = retrieve_job_status(accession)
while job_status != 'COMPLETED':
print 'waiting for %s, Job status: %s' % (accession, job_status)
if job_status == 'FAILED':
raise Exception('%s Unload Failed' % accession)
time.sleep(30)
job_status = retrieve_job_status(accession)
def import_geo_platform(geo_acc):
try:
soft_file = download_soft_file(geo_acc)
header, table = parse_soft_file(soft_file)
generate_adf(geo_acc, header, table)
adf_file = os.path.join(settings.ADF_LOAD_DIR,
geo_acc.replace('GPL', 'A-GEOD-'),
geo_acc + '.adf.txt')
print execute_command('magetab_insert_array.pl -f %s -a %s -c' %
(adf_file, geo_acc.replace('GPL', 'A-GEOD-')))
# shutil.copyfile(os.path.join(settings.ADF_LOAD_DIR, geo_acc.replace('GPL', 'A-GEOD-')), )
out, err = execute_command('reset_array.pl -a A-GEOD-%s -c' %
geo_acc.replace('GPL', ''))
if 'error' in out.lower() or 'error' in err.lower():
msg = """Dear Curators,
While trying to execute rest_array.pl for %s the we had the following output:
%s
%s""" % (geo_acc, out, err)
send_email(from_email='AE Automation<*****@*****.**>',
to_emails=['*****@*****.**', '*****@*****.**'],
subject='GEO Array Error ' +
geo_acc.replace('GPL', 'A-GEOD-'),
body=msg)
return
submit_conan_task(accession=geo_acc.replace('GPL', 'A-GEOD-'),
pipeline_name=CONAN_PIPELINES.load_adf)
except Exception, e:
msg = """The following error occurred while importing: %s
%s""" % (geo_acc, str(e))
send_email(from_email='AE Automation<*****@*****.**>',
to_emails=['*****@*****.**'],
subject='Platform imported',
body=msg)
def change_to_private(email_body, email_address):
ids = geo_email_parse(email_body)
for geo_id, ae_id in ids.items():
check_url = 'http://ves-hx-69.ebi.ac.uk:8983/solr/analytics/select?q=experimentAccession%3A' + ae_id + '&rows=0&wt=json&indent=true'
# print check_url
r = requests.get(check_url)
# print colored.blue(r.content)
if r.json()['response']['numFound'] > 0:
ATLAS_EXPERIMENTS.append(ae_id)
print 'geo id: ', geo_id, ' AE id: ', ae_id
status = retrieve_experiment_status(ae_id)
remove_geo_accession(geo_id)
if status is None:
print 'Not Imported'
continue
update_experiment_status(ae_id)
if status not in ([
'Checking failed', 'GEO import failed', 'Validation failed',
None
]):
exp = retrieve_study_id_by_acc(ae_id)
if exp and len(exp) > 0:
print colored.green('%s found and unloading starting' % ae_id)
plantain, plantain_ssh = retrieve_plantain_connection()
plantain.send(
'dos2unix /ebi/microarray/home/arrayexpress/ae2_production/data/EXPERIMENT/GEOD/%s*.txt\n'
% ae_id)
wait_execution(plantain)
# conan = ConanPage(url=settings.CONAN_URL)
# conan.login(login_email=email_address)
# conan.unload_experiment(ae_id)
submit_conan_task(accession=ae_id,
pipeline_name=CONAN_PIPELINES.unload)
print '%s unload submitted to Conan' % ae_id
else:
print colored.blue(
'%s not in AE database. It might has been unloaded before.'
% ae_id)
else:
print colored.yellow('%s status is: %s. No need to unload' %
(ae_id, status))
if ATLAS_EXPERIMENTS:
print 'sending email for ', ATLAS_EXPERIMENTS
body = """Dear %s,
The following GEO experiments are turned into private as requested by GEO, and they are found in Atlas.
=======================================================================================================
%s
This email was sent automatically from the AE Automation Tool.
Thank You!
AE Automation Tool.""" % (settings.ATLAS_CONTACT['name'],
'\n'.join(ATLAS_EXPERIMENTS))
send_email(from_email=settings.AUTOMATION_EMAIL,
to_emails=[settings.ATLAS_CONTACT['email']],
subject='GEO to Private Notification',
body=body)
print colored.green('EMAIL SENT')
from utils.conan.conan import submit_conan_task
__author__ = 'Ahmed G. Ali'
experiments = """E-GEOD-19824
E-GEOD-32202
E-GEOD-38612
E-GEOD-48851
E-GEOD-49090
E-GEOD-49950
E-GEOD-51304
E-GEOD-58958
E-GEOD-58974
E-GEOD-59637
E-GEOD-61830
E-GEOD-63355
E-GEOD-63512""".split('\n')
for e in experiments:
# conan = ConanPage(url=settings.CONAN_URL)
# conan.login(login_email=settings.CONAN_LOGIN_EMAIL)
print e
# conan.unload_experiment(e.strip())
submit_conan_task(accession=e.strip(), pipeline_name=CONAN_PIPELINES.unload)
def reload_experiment(accession):
exp_path = os.path.join(settings.EXPERIMENTS_PATH, 'ERAD', accession,
'atlas')
if os.path.exists(exp_path):
return
global RED_FLAG, REPORT
skip_unload = False
unload_failed = False
task = retrieve_task(accession)
unload_id = None
if task and task.state == 'FAILED':
unload_id = task.id
print colored.green(
'%s found failed in unloading from a previous trial. Trying to load!'
% accession)
abort_task(task)
skip_unload = True
unload_failed = True
# conan = ConanPage(url=settings.CONAN_URL)
# conan.login(login_email=settings.CONAN_LOGIN_EMAIL)
if not skip_unload:
# conan.unload_experiment(accession)
submit_conan_task(accession=accession,
pipeline_name=CONAN_PIPELINES.unload)
task = retrieve_task(accession)
unload_id = task.id
while task.state != 'COMPLETED':
if task.state == 'FAILED':
print colored.blue('%s Unload Failed\nTrying to load' %
accession,
bold=True)
abort_task(task)
unload_failed = True
break
# raise Exception('%s Unload Failed' % accession)
time.sleep(30)
task = retrieve_task(accession)
load = wait_for_ae_export(accession)
if load:
# conan.load_experiment(accession)
submit_conan_task(accession=accession,
pipeline_name=CONAN_PIPELINES.load)
task = retrieve_task(accession)
load_id = task.id
while task.state != 'COMPLETED':
if task.state == 'FAILED':
print colored.blue('%s Loading Failed. ' % accession,
bold=True)
if unload_failed:
print colored.red(
'%s Failed in both unloading and loading' % accession,
bold=True)
abort_task(task)
with LOCK:
RED_FLAG.append(
colored.red((
'''%s Failed in both unloading and loading. Check below URLs:
Unload: %s/summary/%s
Load: %s/summary/%s''' %
(accession, settings.CONAN_URL, unload_id,
settings.CONAN_URL, load_id)).replace(
'//', '/'),
bold=True))
REPORT[accession] += (
'''Failed in both unloading and loading. Check below URLs:
Unload: %s/summary/%s
Load: %s/summary/%s''' %
(settings.CONAN_URL, unload_id, settings.CONAN_URL,
load_id)).replace('//', '/')
return
time.sleep(30)
task = retrieve_task(accession)
with LOCK:
REPORT[accession] += 'Reloaded successfully'