def create_one_time_hash(conn, action, user_id, params):
sql = '''
update one_time_actions set expires = now() + interval '30' day where
user_id=%(user_id)s and action=%(action)s and params=%(params)s
returning id, hash
'''
vals = {'user_id': user_id, 'action': action, 'params': params}
recs = query_database(conn, sql, vals, commit=True, returning=True)
if recs and len(recs) > 0 and len(recs[0]) > 0:
return recs[0][0], recs[0][1]
hashval = uuid.uuid4().hex
sql = '''
insert into one_time_actions(hash,user_id,action,params,expires)
values (%(hashval)s, %(user_id)s, %(action)s, %(params)s, current_timestamp + interval '30' day)
returning id
'''
vals.update({'hashval': hashval})
recs = query_database(conn, sql, vals, commit=True, returning=True)
if not recs[0][0]:
raise Exception("Error getting id")
id_ = int(recs[0][0])
log.debug('One-time hash id: {0} and hash: {1}'.format(id_, hashval))
return id_, hashval
def setup_workdir(cfg):
if cfg['user_workdir'] and len(cfg['workdir']) > 0:
wd = cfg['workdir']
log.info('Using previous working directory (will not process)')
log.info('Directory is: ' + wd)
else:
s = cfg['proc_name'] + '_' + str(os.getpid()) + '_' + random_string(6)
# Hack to avoid creating a bunch of pointless directories
if 'notify' in s:
return
wd = os.path.join(cfg['workdir'], s)
cfg['workdir'] = wd
log.debug('Workdir is: ' + wd)
if os.path.isdir(wd):
log.warn('Working directory already exists! Removing...')
shutil.rmtree(wd)
log.info('Creating work directory: ' + wd)
os.mkdir(wd)
# Some of the processes are location dependent!
os.chdir(wd)
def send_email(
to_address, subject, body, from_address="no-reply@asf-hyp3", retries=0,
maximum_retries=0, mime_type="plain"):
"""Send an email and return whether the email was successfully sent.
We also retry sending the email if something went wrong the first
time, with the maximum number of retries configurable in the
arguments. This method only supports sending plain text emails.
"""
if retries > maximum_retries:
log.critical(
"Notification failed permanently (maximum retries reached)",
)
return False, None
if retries == 0:
log.info("Sending email")
else:
log.info("Retrying email")
smtp = smtplib.SMTP("localhost")
msg = MIMEMultipart('related')
msg["Subject"] = subject
msg["From"] = from_address
msg["To"] = to_address
msg.preamble = 'This is a multi-part message in MIME format.'
msgAlt = MIMEMultipart('alternative')
msg.attach(msgAlt)
msgText = MIMEText('HyP3 product notification email')
msgAlt.attach(msgText)
msgText = MIMEText(body)
msgText.replace_header('Content-Type', 'text/html')
msgAlt.attach(msgText)
log.debug("Sending email from {0} to {1}".format(from_address, to_address))
bcc_address = []
bcc = get_config('general', 'bcc', default='')
if len(bcc) > 0:
bcc_address += bcc.split(',')
log.debug("Bcc: " + str(bcc_address))
try:
smtp.sendmail(from_address, [to_address] + bcc_address, msg.as_string())
except smtplib.SMTPException as e:
msg = str(e)
log.error("Failed to notify user: "******"Notification failed permanently (maximum retries reached)")
return False, msg
return send_email(to_address, subject, body, from_address, retries + 1, maximum_retries)
smtp.quit()
return True, None
def add_instance_record(cfg, conn):
log.debug('Adding instance record')
instance_record = {}
if cfg['proc_node_type'] != 'CLOUD':
instance_record['instance_id'] = socket.gethostname()
instance_record['instance_type'] = 'on-prem'
else:
instance_record['instance_id'] = get_instance_id()
instance_record['instance_type'] = get_instance_type()
instance_record['local_queue_id'] = cfg['id']
cfg['instance_record'] = instance_record
try:
instance_record_sql = '''
insert into instance_records (instance_id, local_queue_id, start_time, instance_type)
values (%(instance_id)s, %(local_queue_id)s, current_timestamp, %(instance_type)s)
'''
query_database(conn=conn,
query=instance_record_sql,
params=instance_record,
commit=True)
except Exception:
log.exception("Instance record could not be inserted")
else:
log.info("Instance record of instance %s and job %s inserted",
instance_record['instance_id'],
instance_record['local_queue_id'])
def check_stopfile(cfg, stopfile):
if os.path.isfile(stopfile):
log.info('Found stopfile: ' + stopfile)
log.debug('Removing stopfile: ' + stopfile)
os.remove(stopfile)
log.info('Stopping')
cleanup_lockfile(cfg)
sys.exit(0)
def is_config(section, key, config_file=None):
if hyp3proclib.default_cfg is None:
log.debug('Config keys requested from uninitialized config file!')
init_config(config_file=config_file)
if hyp3proclib.default_cfg.has_section(section) \
and hyp3proclib.default_cfg.has_option(section, key):
return is_yes(hyp3proclib.default_cfg.get(section, key))
return False
def get_config(section, key, default=None, config_file=None):
if hyp3proclib.default_cfg is None:
log.debug('Config keys requested from uninitialized config file!')
init_config(config_file=config_file)
if hyp3proclib.default_cfg.has_section(section) \
and hyp3proclib.default_cfg.has_option(section, key):
return hyp3proclib.default_cfg.get(section, key)
else:
if default is None:
log.error('No config value for: ' + section + '/' + key)
return default
def run(self):
self.cfg = setup(self.proc_name,
cli_args=self.cli_args,
sci_version=self.sci_version)
with manage_instance_and_lockfile(self.cfg):
total = self.cfg['num_to_process']
log.info('Starting')
log.debug('Processing {0} products.'.format(total))
self._process_all(total)
log.info('Done')
def notify_user_failure(cfg, conn, msg):
if cfg['notify_fail'] is False:
log.info('Notifications for failures not turned on.')
return
log.debug('Preparing to notify user of processing failure')
username, email, wants_email, subscription_name, process_name = get_user_info(cfg, conn)
if wants_email:
log.debug('Notifying {0}...'.format(username))
message = "Hi, {0}\n\n".format(username)
if cfg['sub_id'] > 0:
message += "Your subscription '{0}' attempted to process a product but failed.\n\n".format(subscription_name)
subject = "[{0}] Failed processing for subscription '{1}'".format(cfg['subject_prefix'], subscription_name)
else:
message += "Your one-time '{0}' processing request failed.\n\n".format(process_name)
subject = "[{0}] Failed one-time processing for '{1}'".format(cfg['subject_prefix'], process_name)
# if len(msg.strip())>0:
# message += "\n" + "Captured error message:\n" + msg + "\n\n"
if 'granule_url' in cfg and len(cfg['granule_url']) > 0:
message += "You can download the original data here:<br>" + cfg['granule_url'] + "<br>"
if cfg['other_granule_urls'] is not None:
for url in cfg['other_granule_urls']:
message += url + "<br>"
if "email_text" in cfg and len(cfg["email_text"]) > 0:
message += "\n" + cfg["email_text"] + "\n\n"
else:
message += "\n"
if cfg['sub_id'] > 0:
param = str(cfg['sub_id'])
id_, hashval = create_one_time_hash(conn, 'disable_subscription', cfg['user_id'], param)
message += "Disable this subscription:\n" \
"https://api.hyp3.asf.alaska.edu/onetime/disable_subscription?id="+str(id_)+"&key="+hashval+"\n\n"
# message += "Captured processing info:\n\n" + cfg['log']
queue_email(conn, cfg['id'], email, subject, message)
else:
log.info("Email will not be sent to user {0} due to user preference".format(username))
def update_instance_record(cfg, conn):
if 'instance_record' in cfg:
instance_record = cfg['instance_record']
try:
instance_record_sql = 'update instance_records set end_time=current_timestamp where (instance_id=%(instance_id)s and local_queue_id=%(local_queue_id)s);'
query_database(conn=conn,
query=instance_record_sql,
params=instance_record,
commit=True)
except Exception:
log.exception(
"Instance record for instance %s and job %s could not be updated with job completion time",
instance_record['instance_id'],
instance_record['local_queue_id'])
else:
log.info(
"Instance record for instance %s and job %s had end_time updated with job completion time",
instance_record['instance_id'],
instance_record['local_queue_id'])
else:
log.debug('No instance record found to update')
def send_queued_emails():
with get_db_connection('hyp3-db') as conn:
sql = '''
select id, local_queue_id, recipients, subject, message, attachment_filename, attachment, mime_type
from email_queue where status = 'QUEUED'
'''
recs = query_database(conn, sql)
if len(recs) == 0:
log.info('No emails to send')
for r in recs:
if r and r[0] and r[2] and len(r[2]) > 0:
id_ = int(r[0])
lqid = None
if r[1] is not None:
lqid = int(r[1])
to = r[2]
subject = r[3]
body = r[4]
mime_type = "plain"
if r[7] is not None:
mime_type = r[7]
if mime_type == "text":
mime_type = "plain"
log.info('Emailing ' + to + ' for lqid: ' + str(lqid))
log.debug('Subject: ' + subject)
ok, msg = send_email(to, subject, body, mime_type=mime_type)
if ok:
status = 'SENT'
else:
status = 'FAILED'
log.debug('Updating status to ' + status)
sql = "update email_queue set status = %(status)s, system_message = %(msg)s, processed_time = current_timestamp where id = %(id)s"
query_database(conn, sql, {'status': status, 'msg': msg, 'id': id_}, commit=True)
def find_rtc_zip(dir_, orbit):
log.debug('Orbit: ' + orbit)
rtc_zip = find_in_dir(dir_, [".zip", "AP_", orbit])
log.info("Found RTC zip: " + rtc_zip)
return rtc_zip
def load_all_general_config(cfg, config_file=None):
if hyp3proclib.default_cfg is None:
log.debug('Config keys requested from uninitialized config file!')
init_config(config_file=config_file)
for k in dict(hyp3proclib.default_cfg.items('general')):
cfg[k] = hyp3proclib.default_cfg.get('general', k)
def process_insar(cfg, n):
try:
log.info('Processing ISCE InSAR pair "{0}" for "{1}"'.format(cfg['sub_name'], cfg['username']))
g1, g2 = earlier_granule_first(cfg['granule'], cfg['other_granules'][0])
list_file = 'list.csv'
write_list_file(os.path.join(cfg['workdir'], list_file), g1, g2)
d1 = g1[17:25]
d2 = g2[17:25]
delta = (datetime.datetime.strptime(d2, '%Y%m%d')-datetime.datetime.strptime(d1, '%Y%m%d')).days
ifm_dir = d1 + '_' + d2
cfg['ifm'] = ifm_dir
log.debug('IFM dir is: ' + ifm_dir)
sd1 = d1[0:4]+'-'+d1[4:6]+'-'+d1[6:8]
sd2 = d2[0:4]+'-'+d2[4:6]+'-'+d2[6:8]
cfg["email_text"] = "This is a {0}-day InSAR pair from {1} to {2}.".format(delta, sd1, sd2)
subswath = get_extra_arg(cfg, "subswath", "0")
if subswath == "0":
process(cfg, 'procAllS1StackISCE.py', ["-90", "90", "-180", "180", "-f", list_file, "-d"])
else:
process(cfg, 'procS1StackISCE.py', ["-f", list_file, "-d", "-s", subswath])
subdir = os.path.join(cfg['workdir'], 'PRODUCT')
if not os.path.isdir(subdir):
log.info('PRODUCT directory not found: ' + subdir)
log.error('Processing failed')
raise Exception("Processing failed: PRODUCT directory not found")
else:
looks = get_looks(subdir)
out_name = build_output_name_pair(g1, g2, cfg['workdir'], looks + "-iw" + subswath + cfg['suffix'])
log.info('Output name: ' + out_name)
out_path = os.path.join(cfg['workdir'], out_name)
zip_file = out_path + '.zip'
if os.path.isdir(out_path):
shutil.rmtree(out_path)
if os.path.isfile(zip_file):
os.unlink(zip_file)
cfg['out_path'] = out_path
# clip_tiffs_to_roi(cfg, conn, product)
log.debug('Renaming '+subdir+' to '+out_path)
os.rename(subdir, out_path)
find_browses(cfg, out_path)
cfg['attachment'] = find_phase_png(out_path)
add_esa_citation(g1, out_path)
zip_dir(out_path, zip_file)
cfg['final_product_size'] = [os.stat(zip_file).st_size, ]
cfg['original_product_size'] = 0
with get_db_connection('hyp3-db') as conn:
record_metrics(cfg, conn)
upload_product(zip_file, cfg, conn)
success(conn, cfg)
except Exception as e:
log.exception('Processing failed')
log.info('Notifying user')
failure(cfg, str(e))
cleanup_workdir(cfg)
log.info('Done')
def find_subswath(dir_):
sub = next(os.walk(dir_))[1][0]
log.debug('Subswath dir: ' + sub)
return sub
def notify_user(product_url, queue_id, cfg, conn):
"""Email a user notifying them of a finished product.
Takes the name of a finished product, download link for the finished
product, subscription ID for the product, the configuration
parameters, and a database connection, and emails the user to notify
them that the product has been finished and provide them with the
download links for both the finished product and the original
granule.
This function return True upon success and False upon failure.
"""
username, email, wants_email, subscription_name, process_name = get_user_info(cfg, conn)
if cfg['sub_id'] > 0:
title = "A new '{0}' product for your subscription '{1}' is ready.".format(process_name, subscription_name)
subject = "[{0}] New product for subscription '{1}'".format(cfg['subject_prefix'], subscription_name)
else:
title = "A new product for your '{0}' one-time processing request has been generated.".format(process_name)
subject = "[{0}] New {1} product available".format(cfg['subject_prefix'], process_name)
message = get_email_header(title)
message += "<p>Hello HyP3-User!"
message += "<p>" + title + "\n"
if 'description' in cfg and cfg['description'] and len(cfg['description']) > 0:
message += "<p>" + escape(cfg['description'], quote=False).replace('\n', '<br>') + "<br>\n"
if process_name != "Notify Only":
message += '<p>You can download it here:<br><a href="{0}">{1}</a><br><br>\n'.format(product_url, cfg['filename'])
if 'browse_url' in cfg and cfg['browse_url'] is not None and len(cfg['browse_url']) > 0:
message += '<center><a href="{0}"><img src="{1}" width="80%" border="0"/></a></center><br>\n'.format(cfg['browse_url'], cfg['browse_url'])
if 'final_product_size' in cfg:
sz = cfg['final_product_size'][0]
mb = float(sz)/1024.0/1024.0
message += "<p>Size: %.2f MB<br><br>\n" % mb
message += "You can find all of your products at the HyP3 website:<br>{0}/products<br>\n".format(cfg['hyp3_product_url'])
if 'granule_url' in cfg and len(str(cfg['granule_url'])) > 0 and 'Subscription: ' not in str(cfg['granule_url']):
message += "<p>You can download the original data from the ASF datapool here:<br>" + urlify(cfg['granule_url']) + "<br>\n"
if 'other_granule_urls' in cfg and cfg['other_granule_urls'] is not None:
for url in cfg['other_granule_urls']:
message += urlify(url) + "<br>\n"
if 'SLC' in cfg['granule']:
message += '<p>View this stack in the ASF baseline tool:<br>'
message += 'http://baseline.asf.alaska.edu/#baseline?granule={0}\n'.format(cfg['granule'])
else:
message += "<p>You can download it here:<br>" + urlify(product_url) + "<br>"
if "email_text" in cfg and len(cfg["email_text"]) > 0:
message += "<p>" + cfg["email_text"] + "<br>"
if 'process_time' in cfg:
message += process_name + " processing time: " + str(datetime.timedelta(seconds=int(cfg['process_time']))) + "<br>\n"
if cfg['sub_id'] > 0:
param = str(cfg['sub_id'])
id_, hashval = create_one_time_hash(conn, 'disable_subscription', cfg['user_id'], param)
message += "<p>Done with this subscription? Disable it with this link:<br>" \
"https://api.hyp3.asf.alaska.edu/onetime/disable_subscription?id="+str(id_)+"&key="+hashval+"<br><br>\n"
message += get_email_footer()
# message += "Hostname: " + socket.gethostname() + "\n"
if wants_email:
log.info('Emailing: ' + email)
queue_email(conn, cfg['id'], email, subject, usr(message, username))
else:
log.info("Email will not be sent to user {0} due to user preference".format(username))
bcc = get_config('general', 'bcc', default='')
if len(bcc) > 0:
# We only have to do the first one, the rest will be bcc'ed :)
addr = bcc.split(',')[0]
log.debug('Queueing email for BCC user: '******'id'], addr, subject, usr(message, username))