def main(date):
# Capture the start time in order to caluclate total processing time
startTime = time.time()
startDate, endDate = get_first_and_last_days_of_month(date)
fitDir = startDate.strftime(helper.FITACF_DIR_FMT)
netDir = startDate.strftime(helper.NETCDF_DIR_FMT)
os.makedirs(fitDir, exist_ok=True)
os.makedirs(netDir, exist_ok=True)
# fitACF 2.5
if DOWNLOAD_25_SOURCE_FILES:
download_fitacfs_from_globus(fitDir, startDate, 'fitacf_25')
convert_fitacf_to_netcdf(startDate, endDate, fitDir, netDir, 2.5)
remove_converted_files(fitDir)
# fitACF 3.0 (speckled)
if DOWNLOAD_30_SOURCE_FILES:
download_fitacfs_from_globus(fitDir, startDate, 'fitacf_30')
convert_fitacf_to_netcdf(startDate, endDate, fitDir, netDir, 3.0)
remove_converted_files(fitDir)
os.rmdir(fitDir)
totalTime = helper.getTimeString(time.time() - startTime)
emailSubject = '"RawACF Download and Conversion Complete"'
emailBody = '"Finished downloading and converting {month} fitACF data\nTotal time: {time}"'.format(
month=startDate.strftime('%Y/%m'), time=totalTime)
helper.send_email(emailSubject, emailBody)
def user_register(data, stage, app_url):
team_id = data['ctf_time_team_id']
team_id = None if team_id == '' else int(team_id)
email = data['email']
password = data['password']
team_name = data['team_name']
with psql_connection(SECRETS['DB_PASSWORD']) as psql:
with psql.cursor() as cursor:
LOGGER.info('USER REGISTER {}'.format(email))
try:
cursor.execute(
'INSERT INTO users VALUES (DEFAULT, now(), '
'NULL, %s, crypt(%s, gen_salt(\'bf\', 10)), '
'%s, %s)', (email, password, team_name, team_id))
except psycopg2.IntegrityError as exception:
if 'email' in exception.diag.constraint_name:
return api_response(409, 'duplicate email')
return api_response(409, 'duplicate team name')
cursor.execute('SELECT id FROM users where email=%s;', (email, ))
user_id = cursor.fetchone()[0]
confirmation_id = str(uuid.uuid4())
cursor.execute('INSERT INTO confirmations VALUES (%s, %s);',
(confirmation_id, user_id))
psql.commit()
confirmation_url = '{}/#/confirm/{}'.format(app_url, confirmation_id)
body = 'Please confirm your account creation:\n\n{}\n'.format(
confirmation_url)
send_email('OOO Account Registration <*****@*****.**>',
email,
'[OOO] Please Confirm Your Registration',
body,
stage=stage)
return api_response(201)
def user_confirm(data, stage):
confirmation_id = data["id"]
with psql_connection(DB_PASSWORD, "scoreboard") as psql:
with psql.cursor() as cursor:
LOGGER.info("CONFIRM: {}".format(confirmation_id))
cursor.execute("SELECT user_id FROM confirmations where id=%s;",
(confirmation_id, ))
response = cursor.fetchone()
if not response:
return api_response(
409,
"invalid confirmation or confirmation already completed")
user_id = response[0]
cursor.execute("DELETE FROM confirmations where id=%s;",
(confirmation_id, ))
cursor.execute(
"UPDATE users SET date_confirmed=now() where id=%s;",
(user_id, ))
cursor.execute("SELECT email FROM users where id=%s;", (user_id, ))
email = cursor.fetchone()[0]
psql.commit()
LOGGER.info("EMAIL: {}".format(email))
body = ("Your registration to DEF CON CTF Quals is complete.\n\n"
"https://scoreboard.oooverflow.io/\n")
send_email(
"OOO Account Registration <*****@*****.**>",
email,
"[OOO] Registration Complete",
body,
stage=stage,
)
return api_response(200, "confirmation complete")
def failedToGrabData(year):
# Send an email and end the script if rsync didn't succeed
emailSubject = '"Unsuccessful attempt to grab {0} Globus Data"'.format(
year)
emailBody = '"Tried to copy {0} from Globus {1} times, but did not succeed. \nSee logfile for more details."'.format(
year, MAX_NUM_TRIES)
helper.send_email(emailSubject, emailBody)
def modify_user():
password = request.form.get("password")
passwordHash = generate_password_hash(password)
if session['nivel2'] != '4':
if request.form.get("nivel") == '4':
mail_body = "This is message from"
mail_html = """<html>
<p style='margin: 0px 0px 8px; text-overflow: ellipsis; overflow-wrap: break-word; color: rgb(23, 43, 77); font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Noto Sans", Ubuntu, "Droid Sans", "Helvetica Neue", sans-serif; font-size: 14px; font-style: normal; font-variant-ligatures: normal; font-variant-caps: normal; font-weight: 400; letter-spacing: normal; orphans: 2; text-align: start; text-indent: 0px; text-transform: none; white-space: normal; widows: 2; word-spacing: 0px; -webkit-text-stroke-width: 0px; background-color: rgb(255, 255, 255); text-decoration-thickness: initial; text-decoration-style: initial; text-decoration-color: initial;'>Desde la Secretaría de Modernización comunicamos que fue habilitado para el segundo nivel de la capacitación en la Plataforma GDE Training.</p>
<p style='margin: 0px 0px 8px; text-overflow: ellipsis; overflow-wrap: break-word; color: rgb(23, 43, 77); font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Noto Sans", Ubuntu, "Droid Sans", "Helvetica Neue", sans-serif; font-size: 14px; font-style: normal; font-variant-ligatures: normal; font-variant-caps: normal; font-weight: 400; letter-spacing: normal; orphans: 2; text-align: start; text-indent: 0px; text-transform: none; white-space: normal; widows: 2; word-spacing: 0px; -webkit-text-stroke-width: 0px; background-color: rgb(255, 255, 255); text-decoration-thickness: initial; text-decoration-style: initial; text-decoration-color: initial;'>Para ingresar al segundo nivel de la capacitacion ingrese a la plataforma <a href="www.gdetraining.com">https://www.gdetraining.com</a> con sus credenciales</p>
<p style='margin: 0px; text-overflow: ellipsis; overflow-wrap: break-word; color: rgb(23, 43, 77); font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Noto Sans", Ubuntu, "Droid Sans", "Helvetica Neue", sans-serif; font-size: 14px; font-style: normal; font-variant-ligatures: normal; font-variant-caps: normal; font-weight: 400; letter-spacing: normal; orphans: 2; text-align: start; text-indent: 0px; text-transform: none; white-space: normal; widows: 2; word-spacing: 0px; -webkit-text-stroke-width: 0px; background-color: rgb(255, 255, 255); text-decoration-thickness: initial; text-decoration-style: initial; text-decoration-color: initial;'>Canales de comunicación para acceder a tutores<br>What app 388 4383563 7-13h días laborables<br>Mail <a href="mailto:[email protected]" style="background-color: transparent; color: rgb(23, 43, 77);">[email protected]</a></p>
</html>"""
mail_subject = "Segundo nivel GDE Training"
mail_recipient = request.form.get("email")
send_email(mail_body, mail_html, mail_subject, mail_recipient)
user_Id = request.form.get('id')
cuil = request.form.get('cuil')
username = request.form.get('username')
password = passwordHash
reparticion = request.form.get('reparticion')
email = request.form.get('email')
nombre = request.form.get('nombre')
apellido = request.form.get('apellido')
nivel = request.form.get("nivel")
modify_user_query(user_Id, cuil, username, password, reparticion, email,
nombre, apellido, nivel)
return redirect(url_for("usersApp.all_users"))
def user_confirm(data, stage):
confirmation_id = data['id']
with psql_connection(SECRETS['DB_PASSWORD']) as psql:
with psql.cursor() as cursor:
LOGGER.info('CONFIRM: {}'.format(confirmation_id))
cursor.execute('SELECT user_id FROM confirmations where id=%s;',
(confirmation_id, ))
response = cursor.fetchone()
if not response:
return api_response(
409, 'invalid confirmation or confirmation'
' already completed')
user_id = response[0]
cursor.execute('DELETE FROM confirmations where id=%s;',
(confirmation_id, ))
cursor.execute(
'UPDATE users SET date_confirmed=now() '
'where id=%s;', (user_id, ))
cursor.execute('SELECT email FROM users where id=%s;', (user_id, ))
email = cursor.fetchone()[0]
psql.commit()
LOGGER.info('EMAIL: {}'.format(email))
body = ('Your registration to DEF CON 2018 CTF Quals is complete.\n\n'
'https://scoreboard.oooverflow.io/\n')
send_email('OOO Account Registration <*****@*****.**>',
email,
'[OOO] Registration Complete',
body,
stage=stage)
return api_response(200, 'confirmation complete')
def convert_rawacf_to_fitacf_and_netcdf(startDate, endDate, rawDir, fitDir, netDir):
raw_to_nc.main(startDate, endDate, rawDir,fitDir,netDir)
dateString = startDate.strftime('%Y/%m')
emailSubject = '"{date} rawACF to netCDF Conversion Successful"'.format(date = dateString)
emailBody = 'Finished converting {date} rawACF files to fitACF and netCDF'.format(date = dateString)
helper.send_email(emailSubject, emailBody)
def get_bug_report(self):
# set gzip header
cherrypy.response.headers['Content-Type'] = 'application/gzip'
# create return tuple
return_tuple = {}
return_tuple['status'] = "failed"
return_tuple['warning'] = ""
return_tuple['error'] = ""
# parse json encoded input
input = helper.convert_dict_values_to_utf8(cherrypy.request.json)
# user language
language = ""
if input.has_key("language") == True:
language = input['language']
# if the user sends a language, which is not german, take the default language setting
if language != "de":
language = Config().get_param("default_language")
# initialize the translator object with the user's choosen language
translator = Translator(language)
# bug report variables
if input.has_key("file_name") == False:
return_tuple['error'] = translator.translate(
"message", "no_bug_report_file_name")
return helper.zip_data(return_tuple)
if input.has_key("bug_report") == False:
return_tuple['error'] = translator.translate(
"message", "no_bug_report_contents")
return helper.zip_data(return_tuple)
# save bug report
try:
bug_report_folder = os.path.join(Config().get_param("logs_folder"),
"client")
if os.path.exists(bug_report_folder) == False:
os.makedirs(bug_report_folder)
file = open(
os.path.join(bug_report_folder,
input['file_name'].split("/")[-1]), 'w')
file.write(input['bug_report'])
file.close()
except IOError as e:
pass
# send mail to the admin
helper.send_email(
"OSMRouter: New bug report", "%s\n\n%s" %
(input['file_name'].split("/")[-1], input['bug_report']))
# convert return_tuple to json and zip it, before returning
return_tuple['status'] = "ok"
return helper.zip_data(return_tuple)
def register_user():
if request.method == "GET":
return render_template("authentication/register.html")
elif request.method == "POST":
nivel = 1
if not request.form.get('cuil'):
return error("Revise el cuil e intente nuevamente")
if not request.form.get('username'):
return error("Revise el username e intente nuevamente")
if not request.form.get('reparticion'):
return error("Revise la reparticion e intente nuevamente")
if not request.form.get('email'):
return error("Revise el email e intente nuevamente")
if not request.form.get('nombre'):
return error("Revise el nombre e intente nuevamente")
if not request.form.get('apellido'):
return error("Revise el apellido e intente nuevamente")
try:
if session["rol"] != 3:
rol = 1
else:
rol = request.form.get('rol')
cuil = request.form.get('cuil'),
username = request.form.get('username'),
password = generate_password_hash(request.form.get('password')),
reparticion = request.form.get('reparticion'),
email = request.form.get('email'),
nombre = request.form.get('nombre'),
apellido = request.form.get('apellido'),
rol = rol,
nivel = nivel
register_query(cuil, username, password, reparticion, email,
nombre, apellido, rol, nivel)
body = "This is message from" + request.form.get("email") + "\r\n"
html = """<html>
<p style='margin: 0px 0px 8px; text-overflow: ellipsis; overflow-wrap: break-word; color: rgb(23, 43, 77); font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Noto Sans", Ubuntu, "Droid Sans", "Helvetica Neue", sans-serif; font-size: 14px; font-style: normal; font-variant-ligatures: normal; font-variant-caps: normal; font-weight: 400; letter-spacing: normal; orphans: 2; text-align: start; text-indent: 0px; text-transform: none; white-space: normal; widows: 2; word-spacing: 0px; -webkit-text-stroke-width: 0px; background-color: rgb(255, 255, 255); text-decoration-thickness: initial; text-decoration-style: initial; text-decoration-color: initial;'>Desde la Secretaría de Modernización comunicamos que fue habilitado para la capacitación en la Plataforma GDE Training.</p>
<p style='margin: 0px 0px 8px; text-overflow: ellipsis; overflow-wrap: break-word; color: rgb(23, 43, 77); font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Noto Sans", Ubuntu, "Droid Sans", "Helvetica Neue", sans-serif; font-size: 14px; font-style: normal; font-variant-ligatures: normal; font-variant-caps: normal; font-weight: 400; letter-spacing: normal; orphans: 2; text-align: start; text-indent: 0px; text-transform: none; white-space: normal; widows: 2; word-spacing: 0px; -webkit-text-stroke-width: 0px; background-color: rgb(255, 255, 255); text-decoration-thickness: initial; text-decoration-style: initial; text-decoration-color: initial;'>Para ingreso <a href="https://www.gdetraining.com/" rel="noreferrer nofollow" style="background-color: transparent; color: rgb(23, 43, 77);" target="_blank">https://www.gdetraining.com/</a><br>Usuario:""" + request.form.get(
'username') + """<br>Clave:""" + request.form.get(
'password') + """</p>
<p style='margin: 0px 0px 8px; text-overflow: ellipsis; overflow-wrap: break-word; color: rgb(23, 43, 77); font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Noto Sans", Ubuntu, "Droid Sans", "Helvetica Neue", sans-serif; font-size: 14px; font-style: normal; font-variant-ligatures: normal; font-variant-caps: normal; font-weight: 400; letter-spacing: normal; orphans: 2; text-align: start; text-indent: 0px; text-transform: none; white-space: normal; widows: 2; word-spacing: 0px; -webkit-text-stroke-width: 0px; background-color: rgb(255, 255, 255); text-decoration-thickness: initial; text-decoration-style: initial; text-decoration-color: initial;'>La capacitación finaliza el 25 de febrero del corriente año, espacio en el cuál deberá completar todas las actividades requeridas.</p>
<p style='margin: 0px; text-overflow: ellipsis; overflow-wrap: break-word; color: rgb(23, 43, 77); font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Noto Sans", Ubuntu, "Droid Sans", "Helvetica Neue", sans-serif; font-size: 14px; font-style: normal; font-variant-ligatures: normal; font-variant-caps: normal; font-weight: 400; letter-spacing: normal; orphans: 2; text-align: start; text-indent: 0px; text-transform: none; white-space: normal; widows: 2; word-spacing: 0px; -webkit-text-stroke-width: 0px; background-color: rgb(255, 255, 255); text-decoration-thickness: initial; text-decoration-style: initial; text-decoration-color: initial;'>Canales de comunicación para acceder a tutores<br>What app 388 4383563 7-13h días laborables<br>Mail <a href="mailto:[email protected]" style="background-color: transparent; color: rgb(23, 43, 77);">[email protected]</a></p>
</html>"""
subject = 'Confirmacion de registro Secretaria de modernizacion'
recipient = request.form.get("email")
try:
send_email(body, html, subject, recipient)
except:
error(
"Hubo un error al enviar el mail. Enviarlo manualmente o revisar el mail"
)
return redirect(url_for("cursos"))
except:
return error("Revise los campos e intente nuevamente")
def save_exchange(buyer, tstamp, fee, data):
""" Save a new SuiExchange entity if not saved yet, key_name is method_order_number.
@param data : {'item_id':'sup1','quantity':1,'price':10,'order_number':'2343-3422','currency':'GBP','method':'GC'}
@exception : db error
"""
m = data['method']
on = data['order_number']
keyname = '%s_%s' % (m, on)
exe = SuiExchange.get_by_key_name(keyname)
if isinstance(buyer, basestring):
ukey = buyer
buyer = helper.query_user_by_key(ukey)
else:
ukey = buyer.key().name()
if exe is None:
exe = SuiExchange(key_name=keyname, user=ukey, xtime=tstamp)
q = int(data['quantity'])
pkg = data['item_id']
exe.points = q * gPackages[pkg.lower()]['pts']
exe.quantity = q
exe.price = float(data['price'])
exe.orderef = '%s' % on
exe.currency = data['currency']
exe.method = data['method']
exe.buyerid = '%s' % data['buyer']
exe.package = pkg
exe.fee = 1.0 * (fee or 1)
taskqueue.add(url='/task/log',
params={
'usr':
ukey,
'act':
'bought %s,%s+%s' % (pkg, buyer.points, exe.points),
'dt':
str(datetime.utcnow())
})
try:
exe.put()
buyer.points += exe.points
buyer.save()
logging.debug('SuiExchange saved, %s, %s' % (m, on))
taskqueue.add(url='/task/dau',
params={
'usr': buyer.key().name(),
'act': 'pay',
'par': pkg,
'qty': str(q)
})
except Exception, e:
logging.exception('SuiExchange.put() failed:%s' % e)
helper.send_email('Suinova test payment failure notification',
str(e))
raise
def download_files_from_globus(rawDir, netDir, date):
# Start Globus Connect Personal and establish connection
# Also allow access to /project/superdarn/data/
subprocess.call('{0} -start -restrict-paths \'rw~/,rw/project/superdarn/data\' &'.format(helper.GLOBUS_PATH), shell=True)
# Initiate the transfer from Globus to APL
subprocess.call('nohup /project/superdarn/software/python-3.8.1/bin/python3 /homes/superdarn/globus/sync_radar_data_globus.py -y {0} -m {1} {2}'.format(date.year, date.month, rawDir), shell=True)
# Stop Globus Connect Personal
subprocess.call('{0} -stop'.format(helper.GLOBUS_PATH), shell=True)
emailSubject = '"{0} rawACF Data Successfully Downloaded"'.format(date.strftime('%Y/%m'))
emailBody = '"{0} rawACF source files have been downloaded. Starting conversion to fitACF and netCDF."'.format(date.strftime('%Y/%m'))
helper.send_email(emailSubject, emailBody)
def profile(web,args=None):
""" Readers apply for authorship or authors update their profile.
Form is submitted and it returns the main page with authors as pageview.
User input will be saved in SuiAuthor entity but SuiUser.role is still 'R'.
An email will be sent to admin who will then review the author and change R to A, and notify the author.
"""
if not web.logged_in:
web.redirect_with_msg('Login required')
return
me = web.user
parms=['title','fname','lname','email','address','job','intro','web','blog','facebook','twitter','linkedin','other','bda']
title,fname,lname,email,address,job,intro,webs,blog,facebook,twitter,linkedin,other,bda=web.get_params(parms)
if me.isAuthor():
a = SuiAuthor.get_by_key_name(me.key().name())
else:
a = SuiAuthor(key_name=me.key().name(),name=me.name)
if title: a.title = title
if fname: a.firstname = fname
if lname: a.lastname = lname
if email: a.email = email
if address: a.address = address
if job: a.job = job
if intro:
ins = intro.replace('"','"').replace('<','<')
a.intro = ins
#a.intro = ''.join(['<p>%s</p>'%ls.strip() for ls in intro.replace('<','<').split('\n')])
lnks = []
if webs: lnks.append('Web: %s'%webs)
if blog: lnks.append('Blog: %s'%blog)
# if facebook: lnks.append('Facebook: %s'%facebook)
if facebook: a.fbpage = facebook
if twitter: lnks.append('Twitter: %s'%twitter)
if linkedin: lnks.append('LinkedIn: %s'%linkedin)
if other: lnks.append('Other: %s'%other)
if lnks:
a.links = '[%s]'%','.join(['"%s"'%s for s in lnks]) #["web:http:..","twitter:",..]
try:
a.put()
if me.role == 'R':
me.role = 'a'
me.save()
msg = APP_MSG % (me.name,me.email or '',facebook,'\r\n'.join(lnks),me.key().name(),bda)
helper.send_email('[Suinova Comics] %s appled for Authorship'%me.name,msg,'*****@*****.**')
else:
me.detach_author()
except Exception,e:
logging.exception(e)
web.redirect_with_msg('Server error, retry later')
return
def main(date):
startTime = time.time()
emailSubject = '"Starting Zenodo Upload"'
emailBody = '"Starting to upload {0} netCDF files to Zenodo"'.format(
date.strftime('%Y-%m'))
helper.send_email(emailSubject, emailBody)
upload_to_zenodo(SANDBOX, date)
totalTime = helper.getTimeString(time.time() - startTime)
emailSubject = '"Finished Zenodo Upload"'
emailBody = '"Finished uploading {0} netCDF files\nTotal time: {1}"'.format(
date.strftime('%Y-%m'), totalTime)
helper.send_email(emailSubject, emailBody)
def get_bug_report(self):
# set gzip header
cherrypy.response.headers['Content-Type'] = 'application/gzip'
# create return tuple
return_tuple = {}
return_tuple['status'] = "failed"
return_tuple['warning'] = ""
return_tuple['error'] = ""
# parse json encoded input
input = helper.convert_dict_values_to_utf8( cherrypy.request.json )
# user language
language = ""
if input.has_key("language") == True:
language = input['language']
# if the user sends a language, which is not german, take the default language setting
if language != "de":
language = Config().get_param("default_language")
# initialize the translator object with the user's choosen language
translator = Translator(language)
# bug report variables
if input.has_key("file_name") == False:
return_tuple['error'] = translator.translate("message", "no_bug_report_file_name")
return helper.zip_data(return_tuple)
if input.has_key("bug_report") == False:
return_tuple['error'] = translator.translate("message", "no_bug_report_contents")
return helper.zip_data(return_tuple)
# save bug report
try:
bug_report_folder = os.path.join( Config().get_param("logs_folder"), "client")
if os.path.exists(bug_report_folder) == False:
os.makedirs(bug_report_folder)
file = open(os.path.join(bug_report_folder, input['file_name'].split("/")[-1]), 'w')
file.write(input['bug_report'])
file.close()
except IOError as e:
pass
# send mail to the admin
helper.send_email("OSMRouter: New bug report", "%s\n\n%s"
% (input['file_name'].split("/")[-1], input['bug_report']) )
# convert return_tuple to json and zip it, before returning
return_tuple['status'] = "ok"
return helper.zip_data(return_tuple)
def feedback(web,args=None):
""" Forward a message to admin.
"""
if not web.logged_in:
web.redirect_with_msg('Login required.')
return
msg = web.get_param('msg')
if not msg:
web.redirect_with_msg('Empty message.')
return
sender = web.user
msg2 = 'Sender: %s (%s), email: %s, on %s (GMT)\n\n%s' % (sender.name,sender.key().name(),sender.email,datetime.utcnow(),msg)
import helper
helper.send_email('User Message via Contact Page',msg2)
web.redirect('/')
def auth_password_request(email):
'''This program requests a password change and sends the user a reset code'''
data = get_data()
valid_email = -1
for user in data['users']:
if user['email'] == email:
valid_email = 1
break
if valid_email == -1:
raise InputError("Invalid Email")
#Create a temporary field for this user called reset_code
#When entered correctly it will be reset
user['reset_code'] = generate_reset_code()
# Format of this message is important for else the subject field won't
# actually be the email subject.
send_email(email, str(user['reset_code']))
return {}
def main():
_configure_logger('email_annonce.log')
logger.info("##### Le Pere Noel de la Fremo #####")
with open(__SECURE_DATA_FILEPATH, 'r') as stream:
secure_data = yaml.safe_load(stream)
# debug
#secure_data['personnes'] = {'nboutin' : {'email':'*****@*****.**'}}
# debug
with open(__EMAIL_TEXT_FILE_PATH, 'r') as file:
body = file.read()
for id_, id_param in secure_data['personnes'].items():
helper.send_email(secure_data['sender_email'], secure_data['sender_pwd'], __EMAIL_OBJECT, body, id_param['email'])
logger.info("Finish")
def main():
date = START_DATE
while date >= END_DATE:
emailSubject = '"PROCESS ALL STARTING"'
emailBody = 'Starting to download and process {0} data'.format(
date.strftime('%Y%m'))
helper.send_email(emailSubject, emailBody)
download_and_process_fitacfs.main(date)
#download_and_process_rawacfs.main(date)
upload_nc_to_zenodo.main(date)
emailSubject = '"PROCESS ALL COMPLETE"'
emailBody = 'Finished downloading and processing {0} data'.format(
date.strftime('%Y%m'))
helper.send_email(emailSubject, emailBody)
date -= relativedelta(months=1)
def accept_author(web, args=None):
""" After reviewing author application, click Accept link and here.
Change SuiUser.role from 'R' to 'A', send this user an email as notification.
"""
web.require_admin()
if not args:
logging.warning('admin/accept_author/None')
web.fail('Invalid author accept call')
return
uid = args[0]
bda = web.request.get('bda')
a = SuiAuthor.get_by_key_name(uid)
if not a:
web.fail('User not recorded')
return
u = SuiUser.get_by_key_name(uid)
if u.role != 'A':
u.role = 'A'
a.confirmed = True
try:
db.put([a, u])
u.recache()
a.recache()
if uid.startswith('fb_'):
lts = WELCOME_LETTER % 'http://apps.facebook.com/suicomics/'
else:
lts = WELCOME_LETTER % 'http://suicomics.appspot.com/'
if bda == '1':
lts += 'gift'
helper.send_email('Application Accepted', lts, a.email)
if bda == '1':
bu = SuiBirthdayUser.get_by_key_name(uid)
if not bu:
bu = SuiBirthdayUser(
key_name=uid, name=u.name, access_token=u.access_token
) #update access_token when user change password
bu.creator = True
bu.put()
web.succeed('Done')
except Exception, e:
logging.exception(e)
web.fail('Accept not successful, please notify admin')
return
def send_all_email(data, secure_data):
subject = 'Le Père Noël de la Frémo: Tirage au sort'
for id_, id_param in data.items():
toaddr = secure_data['personnes'][id_]['email']
# DEBUG
# toaddr = secure_data['receiver_test']
# DEBUG
to_gift_id = id_param['history'][-1]
to_fullname = data[to_gift_id]['fullname']
with open(__BODY, 'r') as file:
body = file.read().format(id_param['fullname'], to_fullname)
logger.debug("{} -> {} ({})".format(id_, to_fullname, to_gift_id))
helper.send_email(secure_data['sender_email'],
secure_data['sender_pwd'], subject, body, toaddr)
def getBasFileList():
# Make sure the BAS server is reachable
if not BASServerConnected():
# Send email if BAS couldn't be reached
emailSubject = '"Unable to reach BAS"'
emailBody = 'Unable to reach BAS after trying for {hours} hours.'.format(hours = RETRY * DELAY / 3600)
helper.send_email(emailSubject, emailBody)
sys.exit('{message}'.format(message = emailBody))
os.makedirs(BAS_FILE_LIST_DIR, exist_ok=True)
# TODO: Get BAS DAT files and Globus raw and dat files
# globus login
# globus ls 'c02cb494-1515-11e9-9f9f-0a06afd4a22e:/chroot/sddata/dat/2006'
# globus ls 'c02cb494-1515-11e9-9f9f-0a06afd4a22e:/chroot/sddata/raw/2005'
year = BAS_START_DATE.year
endYear = BAS_END_DATE.year
while year <= endYear:
print('\nGetting BAS data for {0}\n'.format(year))
filename = '{dir}/{yr}_basRawFiles.txt'.format(dir = BAS_FILE_LIST_DIR, yr = year)
# Get a list of all rawACF files on BAS for the given year and put them in a file
os.system('ssh apl@{bas} ls -R /sddata/raw/{yr}/ > {fname}'.format(bas = helper.BAS_SERVER, yr = year, fname = filename))
# Go through the file line by line and add each line to the appropriate daily text file
with open(filename) as mainFileList:
lines = mainFileList.readlines()
for line in lines:
rawFilename = line.strip()
extension = rawFilename.split('.')[-1]
if not extension == 'bz2':
# This line isn't a rawACF filename
continue
day = rawFilename.split('.')[0]
print('Updating {0} file'.format(day))
radar = rawFilename.split('.')[3]
radarFileList = '{dir}/{d}_{r}.txt'.format(dir = BAS_FILE_LIST_DIR, d = day, r = radar)
with open(radarFileList, "a+") as fp:
fp.write(rawFilename + '\n')
year += 1
def user_register(data, stage, app_url):
team_id = data["ctf_time_team_id"]
team_id = None if team_id == "" else int(team_id)
email = data["email"]
password = data["password"]
team_name = data["team_name"]
with psql_connection(DB_PASSWORD, "scoreboard") as psql:
with psql.cursor() as cursor:
LOGGER.info("USER REGISTER {}".format(email))
try:
cursor.execute(
"INSERT INTO users (id, date_created, date_updated, ctf_time_team_id, "
"email, password, team_name) VALUES (DEFAULT, now(), now(), %s, %s, "
"crypt(%s, gen_salt('bf', 10)), %s)",
(team_id, email, password, team_name),
)
except psycopg2.IntegrityError as exception:
if "email" in exception.diag.constraint_name:
return api_response(409, "duplicate email")
return api_response(409, "duplicate team name")
cursor.execute("SELECT id FROM users where email=%s;", (email, ))
user_id = cursor.fetchone()[0]
confirmation_id = str(uuid.uuid4())
cursor.execute(
"INSERT INTO confirmations (id, user_id) VALUES (%s, %s);",
(confirmation_id, user_id),
)
psql.commit()
confirmation_url = "{}/#/confirm/{}".format(app_url, confirmation_id)
body = "Please confirm your account creation:\n\n{}\n".format(
confirmation_url)
send_email(
"OOO Account Registration <*****@*****.**>",
email,
"[OOO] Please Confirm Your Registration",
body,
stage=stage,
)
return api_response(201)
def accept_author(web,args=None):
""" After reviewing author application, click Accept link and here.
Change SuiUser.role from 'R' to 'A', send this user an email as notification.
"""
web.require_admin()
if not args:
logging.warning('admin/accept_author/None')
web.fail('Invalid author accept call')
return
uid = args[0]
bda = web.request.get('bda')
a = SuiAuthor.get_by_key_name(uid)
if not a:
web.fail('User not recorded')
return
u = SuiUser.get_by_key_name(uid)
if u.role != 'A':
u.role = 'A'
a.confirmed = True
try:
db.put([a,u])
u.recache()
a.recache()
if uid.startswith('fb_'):
lts = WELCOME_LETTER % 'http://apps.facebook.com/suicomics/'
else:
lts = WELCOME_LETTER % 'http://suicomics.appspot.com/'
if bda == '1':
lts += 'gift'
helper.send_email('Application Accepted',lts,a.email)
if bda == '1':
bu = SuiBirthdayUser.get_by_key_name(uid)
if not bu:
bu = SuiBirthdayUser(key_name=uid,name=u.name,access_token=u.access_token) #update access_token when user change password
bu.creator = True
bu.put()
web.succeed('Done')
except Exception,e:
logging.exception(e)
web.fail('Accept not successful, please notify admin')
return
def main():
startTime = time.time()
emailSubject = '"Starting Data Check"'
emailBody = 'Starting BAS vs APL data check'
helper.send_email(emailSubject, emailBody)
getBasFileList()
radarList = helper.get_radar_list()
# TODO: Add DAT file check
date = BAS_START_DATE
data = {}
while date <= BAS_END_DATE:
day = date.strftime('%Y%m%d')
data[day] = []
print('Comparing data between BAS and APL on {d}'.format(d = day))
for radar in radarList:
basDataExists = bas_data(date, radar)
aplDataExists = apl_data(date, radar)
result = get_result(basDataExists, aplDataExists)
data[day].append({
'radar': radar,
'result': result
})
date += dt.timedelta(days=1)
outputFile = '{dir}/data_status_{date}.txt'.format(dir = DATA_STATUS_DIR, date = BAS_END_DATE.strftime('%Y%m%d'))
with open(outputFile, 'w') as outfile:
json.dump(data, outfile)
totalTime = helper.getTimeString(time.time() - startTime)
emailSubject = '"Data Status Check Complete"'
emailBody = '"Finished checking BAS data vs APL data.\nTotal check runtime: {0}\nNew JSON file created: {1}"'.format(totalTime, outputFile)
helper.send_email(emailSubject, emailBody)
if REMOVE_BAS_FILE_LIST:
os.system('rm -rf {bas_dir}'.format(bas_dir = BAS_FILE_LIST_DIR))
def main():
startTime = time.time()
emailSubject = '"Starting Data Check"'
emailBody = 'Starting Globus vs Zenodo data check'
helper.send_email(emailSubject, emailBody)
getGlobusFileList()
getZenodoFileList()
radarList = helper.get_radar_list()
date = START_DATE
data = {}
while date <= END_DATE:
day = date.strftime('%Y%m%d')
data[day] = []
print('{0} - Comparing data between Globus and Zenodo on {1}'.format(
time.strftime('%Y-%m-%d %H:%M'), day))
for radar in radarList:
remoteDataExists = getRemoteData(date, radar)
zenodoDataExists = getZenodoData(date, radar)
result = get_result(remoteDataExists, zenodoDataExists)
data[day].append({'radar': radar, 'result': result})
date += dt.timedelta(days=1)
outputFile = '{0}/{1}_data_status.json'.format(DATA_STATUS_DIR,
END_DATE.strftime('%Y%m%d'))
with open(outputFile, 'w') as outfile:
json.dump(data, outfile)
totalTime = helper.getTimeString(time.time() - startTime)
emailSubject = '"Data Status Check Complete"'
emailBody = '"Finished checking Globus vs Zenodo data.\nTotal check runtime: {0}\nNew JSON file created: {1}"'.format(
totalTime, outputFile)
helper.send_email(emailSubject, emailBody)
if REMOVE_REMOTE_FILE_LIST:
os.system('rm -rf {0}'.format(GLOBUS_FILE_LIST_DIR))
def download_fitacfs_from_globus(fitDir, date, pattern):
# Start Globus Connect Personal and establish connection
# Also allow access to /project/superdarn/data/
subprocess.call(
'{0} -start -restrict-paths \'rw~/,rw/project/superdarn/data/fitacf\' &'
.format(helper.GLOBUS_PATH),
shell=True)
# Initiate the Globus -> APL transfer
subprocess.call(
'nohup /project/superdarn/software/python-3.8.1/bin/python3 /homes/superdarn/globus/sync_radar_data_globus.py -y {0} -m {1} -t {2} {3}'
.format(date.year, date.month, pattern, fitDir),
shell=True)
# Stop Globus Connect Personal
#subprocess.call('{0} -stop'.format(helper.GLOBUS_PATH), shell=True)
emailSubject = '"{0} {1} Data Successfully Downloaded"'.format(
date.strftime('%Y/%m'), pattern)
emailBody = '"{0} {1} source files have been downloaded. Starting conversion to netCDF."'.format(
date.strftime('%Y/%m'), pattern)
helper.send_email(emailSubject, emailBody)
def save_exchange(buyer,tstamp,fee,data):
""" Save a new SuiExchange entity if not saved yet, key_name is method_order_number.
@param data : {'item_id':'sup1','quantity':1,'price':10,'order_number':'2343-3422','currency':'GBP','method':'GC'}
@exception : db error
"""
m = data['method']
on = data['order_number']
keyname = '%s_%s' % (m, on)
exe = SuiExchange.get_by_key_name(keyname)
if isinstance(buyer,basestring):
ukey = buyer
buyer = helper.query_user_by_key(ukey)
else:
ukey = buyer.key().name()
if exe is None:
exe = SuiExchange(key_name=keyname,user=ukey,xtime=tstamp)
q = int(data['quantity'])
pkg = data['item_id']
exe.points = q * gPackages[pkg.lower()]['pts']
exe.quantity = q
exe.price = float(data['price'])
exe.orderef = '%s'%on
exe.currency = data['currency']
exe.method = data['method']
exe.buyerid = '%s'%data['buyer']
exe.package = pkg
exe.fee = 1.0 * (fee or 1)
taskqueue.add(url='/task/log',params={'usr':ukey,'act':'bought %s,%s+%s'%(pkg,buyer.points,exe.points),'dt':str(datetime.utcnow())})
try:
exe.put()
buyer.points += exe.points
buyer.save()
logging.debug('SuiExchange saved, %s, %s' % (m, on))
taskqueue.add(url='/task/dau',params={'usr':buyer.key().name(),'act':'pay','par':pkg,'qty':str(q)})
except Exception,e:
logging.exception('SuiExchange.put() failed:%s'%e)
helper.send_email('Suinova test payment failure notification',str(e))
raise
def main(date):
startTime = time.time()
startDate, endDate = get_first_and_last_days_of_month(date)
rawDir = startDate.strftime(RAWACF_DIR_FMT) + '/'
fitDir = startDate.strftime(FITACF_DIR_FMT)
netDir = startDate.strftime(NETCDF_DIR_FMT)
os.makedirs(rawDir, exist_ok=True)
os.makedirs(fitDir, exist_ok=True)
os.makedirs(netDir, exist_ok=True)
if DOWNLOAD_SOURCE_FILES:
download_source_files(rawDir, netDir, startDate)
convert_rawacf_to_fitacf_and_netcdf(startDate, endDate, rawDir, fitDir, netDir)
remove_converted_files(rawDir, fitDir)
totalTime = helper.getTimeString(time.time() - startTime)
emailSubject = '"RawACF Download and Conversion Complete"'
emailBody = '"Finished downloading and converting {month} RawACF data\nTotal time: {time}"'.format(month = startDate.strftime('%Y/%m'), time = totalTime)
helper.send_email(emailSubject, emailBody)
def main():
date = START_DATE
while date >= END_DATE:
emailSubject = '"Starting to update netCDF gflg descriptions"'
emailBody = 'Starting to update netCDF gflg descriptions for {0}'.format(
date.strftime('%Y%m'))
helper.send_email(emailSubject, emailBody)
netcdfDir = date.strftime(helper.NETCDF_DIR_FMT)
fileList = glob.glob(os.path.join(netcdfDir, '*.nc'))
gflgDescription = 'Ground scatter flag for ACF, 1 - ground scatter, 0 - other scatter'
for file in fileList:
fh = nc.Dataset(file, mode='r+')
fh.variables['gflg'].long_name = gflgDescription
fh.close()
emailSubject = '"Finished updated netCDF gflg descriptions"'
emailBody = 'Finished updated netCDF gflg descriptions for {0}'.format(
date.strftime('%Y%m'))
helper.send_email(emailSubject, emailBody)
date -= relativedelta(months=1)
def send_email():
validated = validateRequest(request)
if validated != "validated":
return validated
req_data = request.get_json()
# Get items from the helper
res_data = helper.send_email(req_data)
# Return 404 if item not found
if 'error' in res_data:
response = Response(json.dumps(res_data), status=400, mimetype='application/json')
response.headers["Access-Control-Allow-Origin"] = "*"
return response
# Return response
response = Response(json.dumps(res_data), status=200, mimetype='application/json')
response.headers["Access-Control-Allow-Origin"] = "*"
return response
def ppapprove(web, args=None):
""" Called by PayPal through RETURNURL set in SetExpressCheckout command.
Currently payment is done directly, a better way is to return a review page for the user to confirm payment and
then come back to send DoExpressCheckoutPayment. But the current method is dirty quick.
The L_NAME0 items are not displayed in the sandbox PayPal continue page, not sure whether it's the same on production site.
"""
token = web.request.get('token')
buyerid = web.request.get('PayerID')
pkg = web.request.get('pkg')
ukey = web.request.get('uid')
if pkg == '':
cs = web.request.get('CUSTOM')
if cs.find(':') > 0:
pkg, ukey = cs.split(':')
logging.debug('pay.ppapprove: token=%s,buyerid=%s,pkg=%s,ukey=%s' %
(token, buyerid, pkg, ukey))
pg = gPackages[pkg.lower()]
request = {
'USER': PP_USER,
'PWD': PP_PWD,
'SIGNATURE': PP_SIG,
'VERSION': PP_VER
}
request['PAYERID'] = buyerid
request['TOKEN'] = token
request['PAYMENTACTION'] = 'Sale'
request['AMT'] = '%0.2f' % float(pg['price'])
request['CURRENCYCODE'] = 'GBP'
request['METHOD'] = 'DoExpressCheckoutPayment'
request['CUSTOM'] = '%s:%s' % (ukey, pkg) # does this return?
request['DESC'] = '%s. %s' % (pg['item'], pg['description'])
request['L_NAME0'] = pg['item']
request['L_NUMBER0'] = pg['id']
request['L_DESC0'] = pg['description']
request['L_AMT0'] = request['AMT']
request['L_QTY0'] = '1'
requests = urllib.urlencode(request)
logging.debug('About to send to PayPal: %s' % requests)
try:
result = urlfetch.fetch(
url=PP_ECURL,
payload=requests,
method=urlfetch.POST,
headers={'Content-type': 'application/x-www-form-urlencoded'})
if result.content.find('ACK=Success') >= 0:
data = cgi.parse_qs(result.content)
#token = data['TOKEN'][-1]
payment_status = data['PAYMENTSTATUS'][0]
if payment_status == 'Completed':
data2 = {
'method': 'PP',
'quantity': '1',
'item_id': pkg,
'buyer': buyerid
}
data2['order_number'] = data['TRANSACTIONID'][0]
data2['price'] = data['AMT'][0]
data2['currency'] = data['CURRENCYCODE'][0]
if 'SETTLEAMT' in data:
logging.debug('settlement: %s' % data['SETTLEAMT'][0])
fee = float(data['FEEAMT'][0])
tstamp = datetime_from_timestampz(data['ORDERTIME'][0])
buyer = helper.get_user_by_key(ukey, False)
if buyer is None:
logging.warning('Buyer %s is not user!' % ukey)
helper.send_email(
'Suinova test payment failure notification',
'Buyer %s not found' % ukey)
web.response.out.write('Thank you!')
return
try:
save_exchange(buyer, tstamp, fee, data2)
except Exception:
pass
web.response.out.write(THANKU_PAGE)
else:
logging.warning(
'ppapprove result from DoExpressCheckoutPayment, payment status is %s'
% payment_status)
logging.info(result.content)
web.response.out.write('Not completed')
else:
logging.error(
'ppapprove send DoExpressCheckoutPayment returned: %s' %
result.content)
web.fail(
'Error contacting PayPal, try later, <a href="/">Go back</a>')
except Exception, e:
logging.exception(e)
web.fail('Error contacting PayPal, try later, <a href="/">Go back</a>')
try:
result = urlfetch.fetch(url=GC_REQUEST_URLS[GC_WHICH],
payload=request_xml,
method=urlfetch.POST,
headers=GC_HEADER)
except Exception, e:
logging.exception(e)
web.error(500) #will cause resend to google checkout
try:
if result.content.find('<charge-amount-notification') > 0:
#charged, so save in ds
data = get_google_notification_data(result.content)
if 'original_buyer' not in data:
ers = 'merchange-private-data not available in charge-amount-notification'
logging.error(ers)
helper.send_email('Suinova test payment failure notification',
ers)
web.succeed() #return 200 not to resend to Google Checkout
return
buyer_id = data['original_buyer']
buyer = helper.get_user_by_key(buyer_id,
False) #'gg_%s' % data['buyer'])
if buyer is None:
logging.warning('Buyer %s is not user!' % buyer_id)
helper.send_email('Suinova test payment failure notification',
'Buyer %s not found' % buyer_id)
web.succeed() #return 200 not to resend to Google Checkout
return
gcfee = extract_text_in_tag(result.content, 'total')
if len(gcfee) > 0:
try:
fee = float(gcfee[0])
def scrape_gmail_messages(debug, mailbox_to_scrape, access_oauth_token, access_oauth_token_secret, consumer_key, consumer_secret, app_email_info, error_email_info, admins):
phone_numbers = []
try:
start_datetime = datetime.now()
consumer = oauth.Consumer(consumer_key, consumer_secret)
token = oauth.Token(access_oauth_token, access_oauth_token_secret)
client = oauth.Client(consumer, token)
# Get the email address from Google contacts
resp, xmldoc = client.request('https://www.google.com/m8/feeds/contacts/default/full?max-results=0')
email = get_id(xmldoc)
# Connect with IMAP
url = 'https://mail.google.com/mail/b/%s/imap/' % email
imap = imaplib.IMAP4_SSL('imap.googlemail.com')
imap.authenticate(url, consumer, token)
# Get message count
resp, count = imap.select(mailbox_to_scrape)
# Get only seen messages
resp, messages = imap.search(None, 'SEEN')
messages = messages[0].split()
# TODO: scrape_gmail_messages.update_state(state=(0, len(messages)))
print "Message count: %d" % len(messages)
# Find the phone numbers in each message
for index in range(len(messages)):
if index % 100 == 0:
print 'Message %s/%s (%s numbers so far)' % (index, len(messages), len(phone_numbers))
try:
phone_numbers += find_phone_numbers(imap, messages[index])
# TODO: scrape_gmail_messages.update_state(state=(index + 1, len(messages)))
except:
print 'Error: could not parse message #%s. Skipping.' % index
from traceback import format_exc
print format_exc()
print
imap.logout()
end_datetime = datetime.now()
# Log the process and its performance
if config:
try:
conn = psycopg2.connect(
host = config['postgres']['host'],
port = int(config['postgres']['port']),
user = config['postgres']['username'],
password = config['postgres']['password'],
database = config['postgres']['database'],
)
cur = conn.cursor()
try:
cur.execute("INSERT INTO processed (message_count, phone_number_count, start_time, end_time) VALUES (%s, %s, %s, %s)", (len(messages), len(phone_numbers), start_datetime, end_datetime))
except psycopg2.ProgrammingError:
# Error, reset the connection
conn.rollback()
# Add table and retry
cur.execute("CREATE TABLE processed (message_count integer, phone_number_count integer, start_time timestamp, end_time timestamp);")
cur.execute("INSERT INTO processed (message_count, phone_number_count, start_time, end_time) VALUES (%s, %s, %s, %s)", (len(messages), len(phone_numbers), start_datetime, end_datetime))
print 'Database table "processed" created.'
conn.commit()
cur.close()
except:
print 'Error: could not log performance.'
from traceback import format_exc
print format_exc()
finally:
conn.close()
else:
print 'Processed %s: Phone Numbers = %s, Start = %s, End = %s' % (len(messages), len(phone_numbers), start_datetime.strftime('%m/%d/%Y %I:%M:%S %p'), end_datetime.strftime('%m/%d/%Y %I:%M:%S %p'))
# Send completion email
if app_email_info:
send_email(app_email_info, email, 'Your DijScrape has finished', 'Hi, we are sending this message to inform you that your DijScrape has finished! To see your phone numbers, head back over to http://www.dijscrape.com/')
return phone_numbers
except:
from traceback import format_exc
exc = format_exc()
print exc
if not debug and error_email_info:
send_email(error_email_info, admins, 'DijScrape Task Failed', exc)
return phone_numbers
#send back charge request now
request_xml = GC_CHARGE_SHIP_ORDER % (order_number,currency,total)
logging.debug('gccharge send to goodle checkout: %s'%request_xml)
try:
result = urlfetch.fetch(url=GC_REQUEST_URLS[GC_WHICH],payload=request_xml,method=urlfetch.POST,headers=GC_HEADER);
except Exception,e:
logging.exception(e)
web.error(500) #will cause resend to google checkout
try:
if result.content.find('<charge-amount-notification') > 0:
#charged, so save in ds
data = get_google_notification_data(result.content)
if 'original_buyer' not in data:
ers = 'merchange-private-data not available in charge-amount-notification'
logging.error(ers)
helper.send_email('Suinova test payment failure notification',ers)
web.succeed() #return 200 not to resend to Google Checkout
return
buyer_id = data['original_buyer']
buyer = helper.get_user_by_key(buyer_id,False) #'gg_%s' % data['buyer'])
if buyer is None:
logging.warning('Buyer %s is not user!'%buyer_id)
helper.send_email('Suinova test payment failure notification','Buyer %s not found'%buyer_id)
web.succeed() #return 200 not to resend to Google Checkout
return
gcfee = extract_text_in_tag(result.content,'total')
if len(gcfee)>0:
try:
fee = float(gcfee[0])
except TypeError,e:
logging.error('float(%s) error:%s'%(gcfee[0],e))
def main(
start_time=dt.datetime(2005, 12, 1),
end_time=dt.datetime(2020, 1, 1),
in_dir_fmt='/project/superdarn/data/rawacf/%Y/%m/',
fit_dir_fmt='/project/superdarn/data/fitacf/%Y/%m/',
out_dir_fmt='/project/superdarn/data/netcdf/%Y/%m/',
step=1, # month
skip_existing=True,
fit_ext='*.fit',
):
# Send the output to a log file
original_stdout = sys.stdout
if SAVE_OUTPUT_TO_LOGFILE:
f = open('/homes/superdarn/logs/rawACF_to_netCDF_logs/raw_to_fit_to_net_{startDate}-{endDate}.log'.format(startDate = start_time.strftime("%Y%m%d"), endDate = end_time.strftime("%Y%m%d")), 'w')
sys.stdout = f
rstpath = os.getenv('RSTPATH')
assert rstpath, 'RSTPATH environment variable needs to be set'
hdw_dat_dir = os.path.join(rstpath, 'tables/superdarn/hdw/')
# Running raw to NC
radar_info = get_radar_params(hdw_dat_dir)
run_dir = '/project/superdarn/run/%s' % get_random_string(4)
if in_dir_fmt:
raw_to_fit(start_time, end_time, run_dir, in_dir_fmt, fit_dir_fmt, MAKE_FIT_VERSIONS)
# Loop over fit files in the monthly directories
time = start_time
while time <= end_time:
# Set up directories
out_dir = time.strftime(out_dir_fmt)
print('Trying to make %s' % out_dir)
os.makedirs(out_dir, exist_ok=True)
# Loop over the files
fit_fn_fmt = time.strftime(fit_dir_fmt)
temp = f'fit_fn_fmt: {fit_fn_fmt}'
print(temp)
print(os.path.join(fit_fn_fmt, fit_ext))
fit_fnames = glob.glob(os.path.join(fit_fn_fmt, fit_ext))
print('Processing %i %s files in %s on %s' % (len(fit_fnames), fit_ext, fit_fn_fmt, time.strftime('%Y/%m')))
for fit_fn in fit_fnames:
# Check the file is big enough to be worth bothering with
fn_info = os.stat(fit_fn)
if fn_info.st_size < MIN_FITACF_FILE_SIZE:
print('\n\n%s %1.1f MB\nFile too small - skipping' % (fit_fn, fn_info.st_size / 1E6))
continue
print('\n\nStarting from %s' % fit_fn)
fn_head = '.'.join(os.path.basename(fit_fn).split('.')[:-1])
out_fn = os.path.join(out_dir, '%s.nc' % fn_head)
if os.path.isfile(out_fn):
if skip_existing:
print('%s exists - skipping' % out_fn)
continue
else:
print('%s exists - deleting' % out_fn)
os.remove(out_fn)
# Convert the fitACF to a netCDF
radar_code = os.path.basename(fit_fn).split('.')[1]
radar_info_t = id_hdw_params_t(time, radar_info[radar_code])
status = fit_to_nc(time, fit_fn, out_fn, radar_info_t)
if status == MULTIPLE_BEAM_DEFS_ERROR_CODE:
print('Failed to convert {fitacfFile} because it had multiple beam definitions'.format(fitacfFile = fit_fn))
continue
elif status > 0:
print('Failed to convert {fitacfFile}'.format(fitacfFile = fit_fn))
continue
print('Wrote output to %s' % out_fn)
month = time.strftime('%Y%m')
multiBeamLogDir = time.strftime(helper.FIT_NET_LOG_DIR) + month
multiBeamFile = '{dir}/multi_beam_defs_{m}.log'.format(dir = multiBeamLogDir, m = month)
if os.path.exists(multiBeamFile):
subject = '"Multiple Beam Definitions Found - {date}"'.format(date = time.strftime('%Y/%m'))
body = 'Files with multiple beam definitions have been found. See details in {file}'.format(file = multiBeamFile)
helper.send_email(subject, body)
time = add_months(time, step) # time += dt.timedelta(months=1) doesn't exist
sys.stdout = original_stdout
def ppapprove(web,args=None):
""" Called by PayPal through RETURNURL set in SetExpressCheckout command.
Currently payment is done directly, a better way is to return a review page for the user to confirm payment and
then come back to send DoExpressCheckoutPayment. But the current method is dirty quick.
The L_NAME0 items are not displayed in the sandbox PayPal continue page, not sure whether it's the same on production site.
"""
token = web.request.get('token')
buyerid = web.request.get('PayerID')
pkg = web.request.get('pkg')
ukey = web.request.get('uid')
if pkg == '':
cs = web.request.get('CUSTOM')
if cs.find(':') > 0:
pkg,ukey = cs.split(':')
logging.debug('pay.ppapprove: token=%s,buyerid=%s,pkg=%s,ukey=%s'%(token,buyerid,pkg,ukey))
pg = gPackages[pkg.lower()]
request = {'USER':PP_USER,'PWD':PP_PWD,'SIGNATURE':PP_SIG,'VERSION':PP_VER}
request['PAYERID'] = buyerid
request['TOKEN'] = token
request['PAYMENTACTION'] = 'Sale'
request['AMT'] = '%0.2f'%float(pg['price'])
request['CURRENCYCODE'] = 'GBP'
request['METHOD'] = 'DoExpressCheckoutPayment'
request['CUSTOM'] = '%s:%s' % (ukey,pkg) # does this return?
request['DESC'] = '%s. %s' % (pg['item'],pg['description'])
request['L_NAME0'] = pg['item']
request['L_NUMBER0'] = pg['id']
request['L_DESC0'] = pg['description']
request['L_AMT0'] = request['AMT']
request['L_QTY0'] = '1'
requests = urllib.urlencode(request)
logging.debug('About to send to PayPal: %s'%requests)
try:
result = urlfetch.fetch(url=PP_ECURL,payload=requests,method=urlfetch.POST,headers={'Content-type':'application/x-www-form-urlencoded'})
if result.content.find('ACK=Success')>=0:
data = cgi.parse_qs(result.content)
#token = data['TOKEN'][-1]
payment_status = data['PAYMENTSTATUS'][0]
if payment_status == 'Completed':
data2 = {'method':'PP','quantity':'1','item_id':pkg,'buyer':buyerid}
data2['order_number'] = data['TRANSACTIONID'][0]
data2['price'] = data['AMT'][0]
data2['currency'] = data['CURRENCYCODE'][0]
if 'SETTLEAMT' in data:
logging.debug('settlement: %s'%data['SETTLEAMT'][0])
fee = float(data['FEEAMT'][0])
tstamp = datetime_from_timestampz(data['ORDERTIME'][0])
buyer = helper.get_user_by_key(ukey,False)
if buyer is None:
logging.warning('Buyer %s is not user!'%ukey)
helper.send_email('Suinova test payment failure notification','Buyer %s not found'%ukey)
web.response.out.write('Thank you!')
return
try:
save_exchange(buyer,tstamp,fee,data2)
except Exception:
pass
web.response.out.write(THANKU_PAGE)
else:
logging.warning('ppapprove result from DoExpressCheckoutPayment, payment status is %s'%payment_status)
logging.info(result.content)
web.response.out.write('Not completed')
else:
logging.error('ppapprove send DoExpressCheckoutPayment returned: %s'%result.content)
web.fail('Error contacting PayPal, try later, <a href="/">Go back</a>')
except Exception,e:
logging.exception(e)
web.fail('Error contacting PayPal, try later, <a href="/">Go back</a>')
