def send_ena_report():
global REPORT
if len(REPORT['ERROR']) == len(REPORT['FIXED']) == len(REPORT['CORRECT']) == 0:
print 'No Report to be sent'
return
msg = """This report is about ENA Release Date Check.\n\n"""
if len(REPORT['ERROR']) > 0:
msg += 'The following experiments failed in updating via ENA Webin:\n'
msg += '===========================================================\n\n'
msg += 'ACC | RELEASE DATE | ENA MESSAGE\n'
for i in REPORT['ERROR']:
msg += '%s | %s | %s\n' % (i[0], i[1], i[2])
msg += '\n'
if len(REPORT['FIXED']) > 0:
msg += 'The following experiments were updated on ENA:\n'
msg += '===============================================\n\n'
msg += 'ACC | RELEASE DATE | ENA MESSAGE\n'
for i in REPORT['FIXED']:
msg += '%s | %s | %s\n' % (i[0], i[1], i[2])
msg += '\n'
if len(REPORT['CORRECT']) > 0:
msg += 'The following experiments were correct:\n'
msg += '========================================\n\n'
msg += 'ACC | RELEASE DATE\n'
for i in REPORT['CORRECT']:
msg += '%s | %s\n' % (i[0], i[1])
msg += '\n'
msg += """
This email was sent automatically from the AE Automation Tool.
Thank You!
AE Automation Tool."""
send_email(from_email=settings.AUTOMATION_EMAIL, to_emails=['*****@*****.**'],
subject='ENA Release dates report',
body=msg)
def send_report(total, imported, skipped, error):
msg = """Dear Curators,
Geo Platform import report.
Collected: %d
Imported: %d
Skipped: %d
Errors: %d
""" % (total, imported, skipped, len(error.keys()))
if len(error) > 0:
msg += 'Error are as follows:\n'
for k, v in error.items():
msg += '%s: %s\n' % k, v
send_email(from_email='AE Automation<*****@*****.**>',
to_emails=['*****@*****.**'],
subject='Geo Platform Import Report',
body=msg)
def main():
out, err = execute_command("""sudo -u fg_cur -H sh -c "ssh oy-ena-login-1 ls -d /fire/staging/aexpress/*/" """)
# print out
# print '-' * 30
# print err
# print '-' * 30
dirs = out.split('\n')
removed = []
not_removed = []
for d in dirs:
directory = d.replace('/fire/staging/aexpress/', '').replace('/','')
if directory.startswith('E-MTAB'):
acc = '-'.join(directory.split('-')[:3])
print acc
id = retrieve_study_id_by_acc(acc)
if id:
out1, err1 = execute_command("""sudo -u fg_cur -H sh -c "ssh oy-ena-login-1 rm -rf %s" """%d)
if err1:
not_removed.append('%s:%s' % (directory, err1))
print err1
else:
removed.append(directory)
if removed:
send_email(from_email='AE Automation<*****@*****.**>',
to_emails=[CURATION_EMAIL],
# to_emails=['*****@*****.**'],
subject='/fire/staging Cleaning Report',
body="""Dear Curator,
The directories below have been deleted from /fire/staging. These were found loaded in AE database.
Thanks,
AE Automation Tool.
REMOVED DIRECTORY LIST:
=======================
%s""" % '\n'.join(removed))
if not_removed:
send_email(from_email='AE Automation<*****@*****.**>',
# to_emails=['*****@*****.**', CURATION_EMAIL],
to_emails=['*****@*****.**'],
subject='/fire/staging Error Report',
body="""
ERROR DIRECTORY LIST:
=======================
%s""" % '\n'.join(not_removed))
def manage_threads(accessions):
global REPORT
threads = []
for acc in accessions:
# REPORT[acc] = ''
# reload_experiment(acc)
# break
# print acc
REPORT[acc] = ''
t = threading.Thread(target=reload_experiment, args=(acc, ))
threads.append(t)
t.daemon = False
running = []
while True:
while len(running) <= 10 and threads:
t = threads.pop()
t.start()
running.append(t)
time.sleep(5)
if not running:
break
for t in running:
if not t.is_alive():
print 'removing thread'
running.remove(t)
break
print 'Running Experiments: ', len(running)
print 'pending Experiments: ', len(threads)
time.sleep(30)
if RED_FLAG:
print colored.green('-' * 50 + '\nFAILURE REPORT\n' + '=' * 50)
for flag in RED_FLAG:
print flag
msg = """This report is about ERAD experiments Loaded/updated in ArrayExpress.\n\n"""
for k, v in REPORT.items():
msg += """%s\t%s\n""" % (k, v)
msg += '\n\n' + generate_atlas_curated_report(accessions)
msg += """
This email was sent automatically from the AE Automation Tool.
Thank You!
AE Automation Tool."""
send_email(from_email=settings.AUTOMATION_EMAIL,
to_emails=[settings.CURATION_EMAIL],
subject='ERAD Update Report',
body=msg)
print REPORT
def post(self, request):
out = spring_framework.main()
log(user=request.user, action='Restart Framework', extra={'out': out})
print request.user
print dir(request.user)
print 'USERNAME: '******'AE Automation<*****@*****.**>',
to_emails=[request.user.email],
subject='Framework Restarted',
body=body)
return self.render_json_response({}, 200)
def import_geo_platform(geo_acc):
try:
soft_file = download_soft_file(geo_acc)
header, table = parse_soft_file(soft_file)
generate_adf(geo_acc, header, table)
adf_file = os.path.join(settings.ADF_LOAD_DIR,
geo_acc.replace('GPL', 'A-GEOD-'),
geo_acc + '.adf.txt')
print execute_command('magetab_insert_array.pl -f %s -a %s -c' %
(adf_file, geo_acc.replace('GPL', 'A-GEOD-')))
# shutil.copyfile(os.path.join(settings.ADF_LOAD_DIR, geo_acc.replace('GPL', 'A-GEOD-')), )
out, err = execute_command('reset_array.pl -a A-GEOD-%s -c' %
geo_acc.replace('GPL', ''))
if 'error' in out.lower() or 'error' in err.lower():
msg = """Dear Curators,
While trying to execute rest_array.pl for %s the we had the following output:
%s
%s""" % (geo_acc, out, err)
send_email(from_email='AE Automation<*****@*****.**>',
to_emails=['*****@*****.**', '*****@*****.**'],
subject='GEO Array Error ' +
geo_acc.replace('GPL', 'A-GEOD-'),
body=msg)
return
submit_conan_task(accession=geo_acc.replace('GPL', 'A-GEOD-'),
pipeline_name=CONAN_PIPELINES.load_adf)
except Exception, e:
msg = """The following error occurred while importing: %s
%s""" % (geo_acc, str(e))
send_email(from_email='AE Automation<*****@*****.**>',
to_emails=['*****@*****.**'],
subject='Platform imported',
body=msg)
def change_to_private(email_body, email_address):
ids = geo_email_parse(email_body)
for geo_id, ae_id in ids.items():
check_url = 'http://ves-hx-69.ebi.ac.uk:8983/solr/analytics/select?q=experimentAccession%3A' + ae_id + '&rows=0&wt=json&indent=true'
# print check_url
r = requests.get(check_url)
# print colored.blue(r.content)
if r.json()['response']['numFound'] > 0:
ATLAS_EXPERIMENTS.append(ae_id)
print 'geo id: ', geo_id, ' AE id: ', ae_id
status = retrieve_experiment_status(ae_id)
remove_geo_accession(geo_id)
if status is None:
print 'Not Imported'
continue
update_experiment_status(ae_id)
if status not in ([
'Checking failed', 'GEO import failed', 'Validation failed',
None
]):
exp = retrieve_study_id_by_acc(ae_id)
if exp and len(exp) > 0:
print colored.green('%s found and unloading starting' % ae_id)
plantain, plantain_ssh = retrieve_plantain_connection()
plantain.send(
'dos2unix /ebi/microarray/home/arrayexpress/ae2_production/data/EXPERIMENT/GEOD/%s*.txt\n'
% ae_id)
wait_execution(plantain)
# conan = ConanPage(url=settings.CONAN_URL)
# conan.login(login_email=email_address)
# conan.unload_experiment(ae_id)
submit_conan_task(accession=ae_id,
pipeline_name=CONAN_PIPELINES.unload)
print '%s unload submitted to Conan' % ae_id
else:
print colored.blue(
'%s not in AE database. It might has been unloaded before.'
% ae_id)
else:
print colored.yellow('%s status is: %s. No need to unload' %
(ae_id, status))
if ATLAS_EXPERIMENTS:
print 'sending email for ', ATLAS_EXPERIMENTS
body = """Dear %s,
The following GEO experiments are turned into private as requested by GEO, and they are found in Atlas.
=======================================================================================================
%s
This email was sent automatically from the AE Automation Tool.
Thank You!
AE Automation Tool.""" % (settings.ATLAS_CONTACT['name'],
'\n'.join(ATLAS_EXPERIMENTS))
send_email(from_email=settings.AUTOMATION_EMAIL,
to_emails=[settings.ATLAS_CONTACT['email']],
subject='GEO to Private Notification',
body=body)
print colored.green('EMAIL SENT')
if data_file.execution_error:
report['execution_errors'].append(data_file.execution_error)
shutil.rmtree(local_dir)
except Exception, e:
report['execution_errors'].append(str(e))
v.validation_report = json.dumps(report)
v.status = 'V'
if report['execution_errors']:
v.status = 'P'
send_email(from_email='AE Automation<*****@*****.**>',
to_emails=['*****@*****.**', '*****@*****.**'],
subject='Validation Execution Error',
body="""ID: %s\nError: %s""" %
(req_id, str(report['execution_errors'])))
elif report['file_errors'] or report['integrity_errors'] or report[
'pairs_errors']:
v.status = 'F'
v.save()
print report
if __name__ == '__main__':
from random import randint
from sys import argv
#validate(randint(1, 99999999), 'pairs')
validate(argv[1], argv[2])