def handle_src_build(self):
#cleanup src and target collections
src_clean_archives(noconfirm=True)
target_clean_collections(noconfirm=True)
for config in ('mygene', 'mygene_allspecies'):
t0 = time.time()
p = Popen(['python', '-m', 'databuild.builder', config],
cwd=src_path)
returncode = p.wait()
t = timesofar(t0)
if returncode == 0:
msg = 'Dispatcher: "{}" builder finished successfully with code {} (time: {})'.format(
config, returncode, t)
color = "green"
else:
msg = 'Dispatcher: "{}" builder failed successfully with code {} (time: {})'.format(
config, returncode, t)
color = "red"
print(msg)
if hipchat_msg:
msg += '<a href="{}/log/build/{}">build log</a>'.format(
DATA_WWW_ROOT_URL, config)
hipchat_msg(msg, message_format='html', color=color)
assert returncode == 0, "Subprocess failed. Check error above."
genedoc_merged.send(self)
def check_src_upload(self):
running_processes = self.running_processes_upload
jobs_finished = []
if running_processes:
self.idle = True
print('Dispatcher: {} active job(s)'.format(
len(running_processes)))
print(get_process_info(running_processes))
for src in running_processes:
p = running_processes[src]
returncode = p.poll()
if returncode is None:
p.log_f.flush()
else:
t1 = round(time.time() - p.t0, 0)
d = {
'upload.returncode': returncode,
'upload.timestamp': datetime.now(),
'upload.time_in_s': t1,
'upload.time': timesofar(p.t0),
'upload.logfile': p.logfile,
'upload.status': "success" if returncode == 0 else "failed"
}
mark_upload_done(src, d)
jobs_finished.append(src)
p.log_f.close()
if returncode == 0:
msg = 'Dispatcher: "{}" uploader finished successfully with code {} (time: {})'.format(
src, returncode, timesofar(p.t0, t1=t1))
print(msg)
if hipchat_msg:
msg += '<a href="{}/log/dump/{}">dump log</a>'.format(
DATA_WWW_ROOT_URL, src)
msg += '<a href="{}/log/upload/{}">upload log</a>'.format(
DATA_WWW_ROOT_URL, src)
hipchat_msg(msg, message_format='html', color="green")
source_upload_success.send(self, src_name=src)
else:
msg = 'Dispatcher: "{}" uploader failed with code {} (time: {}s)'.format(
src, returncode, t1)
print(msg)
if hipchat_msg:
hipchat_msg(msg, color="red")
source_upload_failed.send(self, src_name=src)
for src in jobs_finished:
del running_processes[src]
def handle_genedoc_merged(self, **kwargs):
for config in ('mygene', 'mygene_allspecies'):
t0 = time.time()
p = Popen(['python', '-m', 'databuild.sync', config, '-p', '-b'], cwd=src_path)
returncode = p.wait()
t = timesofar(t0)
if returncode == 0:
msg = 'Dispatcher: "{}" syncer finished successfully with code {} (time: {})'.format(config, returncode, t)
else:
msg = 'Dispatcher: "{}" syncer failed successfully with code {} (time: {})'.format(config, returncode, t)
print(msg)
if hipchat_msg:
msg += '<a href="http://su07:8000/log/sync/{}">sync log</a>'.format(config)
hipchat_msg(msg, message_format='html')
assert returncode == 0, "Subprocess failed. Check error above."
def check_src_upload(self):
running_processes = self.running_processes_upload
jobs_finished = []
if running_processes:
self.idle = True
print('Dispatcher: {} active job(s)'.format(len(running_processes)))
print(get_process_info(running_processes))
for src in running_processes:
p = running_processes[src]
returncode = p.poll()
if returncode is None:
p.log_f.flush()
else:
t1 = round(time.time() - p.t0, 0)
d = {
'upload.returncode': returncode,
'upload.timestamp': datetime.now(),
'upload.time_in_s': t1,
'upload.time': timesofar(p.t0),
'upload.logfile': p.logfile,
'upload.status': "success" if returncode == 0 else "failed"
}
mark_upload_done(src, d)
jobs_finished.append(src)
p.log_f.close()
if returncode == 0:
msg = 'Dispatcher: "{}" uploader finished successfully with code {} (time: {})'.format(src, returncode, timesofar(p.t0, t1=t1))
print(msg)
if hipchat_msg:
msg += '<a href="http://su07:8000/log/dump/{}">dump log</a>'.format(src)
msg += '<a href="http://su07:8000/log/upload/{}">upload log</a>'.format(src)
hipchat_msg(msg, message_format='html')
source_upload_success.send(self, src_name=src)
else:
msg = 'Dispatcher: "{}" uploader failed with code {} (time: {}s)'.format(src, returncode, t1)
print(msg)
if hipchat_msg:
hipchat_msg(msg)
source_upload_failed.send(self, src_name=src)
for src in jobs_finished:
del running_processes[src]
def handle_genedoc_merged(self, **kwargs):
for config in ('mygene', 'mygene_allspecies'):
t0 = time.time()
p = Popen(['python', '-m', 'databuild.sync', config, '-p', '-b'],
cwd=src_path)
returncode = p.wait()
t = timesofar(t0)
if returncode == 0:
msg = 'Dispatcher: "{}" syncer finished successfully with code {} (time: {})'.format(
config, returncode, t)
color = "green"
else:
msg = 'Dispatcher: "{}" syncer failed successfully with code {} (time: {})'.format(
config, returncode, t)
color = "red"
print(msg)
if hipchat_msg:
msg += '<a href="{}/log/sync/{}">sync log</a>'.format(
DATA_WWW_ROOT_URL, config)
hipchat_msg(msg, message_format='html', color=color)
assert returncode == 0, "Subprocess failed. Check error above."
def handle_src_build(self):
#cleanup src and target collections
src_clean_archives(noconfirm=True)
target_clean_collections(noconfirm=True)
for config in ('mygene', 'mygene_allspecies'):
t0 = time.time()
p = Popen(['python', '-m', 'databuild.builder', config], cwd=src_path)
returncode = p.wait()
t = timesofar(t0)
if returncode == 0:
msg = 'Dispatcher: "{}" builder finished successfully with code {} (time: {})'.format(config, returncode, t)
else:
msg = 'Dispatcher: "{}" builder failed successfully with code {} (time: {})'.format(config, returncode, t)
print(msg)
if hipchat_msg:
msg += '<a href="http://su07:8000/log/build/{}">build log</a>'.format(config)
hipchat_msg(msg, message_format='html')
assert returncode == 0, "Subprocess failed. Check error above."
genedoc_merged.send(self)
# mark the download starts
doc = {
'_id': 'ucsc',
'timestamp': timestamp,
'data_folder': DATA_FOLDER,
'lastmodified': latest_lastmodified,
'logfile': logfile,
'status': 'downloading'
}
src_dump.save(doc)
t0 = time.time()
download(download_list, no_confirm)
# mark the download finished successfully
_updates = {
'status': 'success',
'time': timesofar(t0),
'pending_to_upload': True # a flag to trigger data uploading
}
src_dump.update({'_id': 'ucsc'}, {'$set': _updates})
if __name__ == '__main__':
try:
main()
hipchat_msg('"ucsc" downloader finished successfully', color='green')
except Exception as e:
import traceback
logging.error("Error while downloading: %s" % traceback.format_exc())
hipchat_msg('"ucsc" downloader failed: %s' % e, color='red')
sys.exit(255)
logfile = os.path.join(DATA_FOLDER, 'pharmgkb_dump.log')
setup_logfile(logfile)
#mark the download starts
doc = {'_id': 'pharmgkb',
'timestamp': timestamp,
'data_folder': DATA_FOLDER,
'lastmodified': lastmodified,
'logfile': logfile,
'status': 'downloading'}
src_dump.save(doc)
t0 = time.time()
download(no_confirm)
#mark the download finished successfully
_updates = {
'status': 'success',
'time': timesofar(t0),
'pending_to_upload': True # a flag to trigger data uploading
}
src_dump.update({'_id': 'pharmgkb'}, {'$set': _updates})
if __name__ == '__main__':
try:
main()
hipchat_msg('"pharmgkb" downloader finished successfully',color='green')
except Exception as e:
import traceback
logging.error("Error while downloading: %s" % traceback.format_exc())
hipchat_msg('"pharmgkb" downloader failed: %s' % e,color='red')
sys.exit(255)