Example #1
0
def main(date):
    # Capture the start time in order to caluclate total processing time
    startTime = time.time()

    startDate, endDate = get_first_and_last_days_of_month(date)

    fitDir = startDate.strftime(helper.FITACF_DIR_FMT)
    netDir = startDate.strftime(helper.NETCDF_DIR_FMT)

    os.makedirs(fitDir, exist_ok=True)
    os.makedirs(netDir, exist_ok=True)

    # fitACF 2.5
    if DOWNLOAD_25_SOURCE_FILES:
        download_fitacfs_from_globus(fitDir, startDate, 'fitacf_25')

    convert_fitacf_to_netcdf(startDate, endDate, fitDir, netDir, 2.5)
    remove_converted_files(fitDir)

    # fitACF 3.0 (speckled)
    if DOWNLOAD_30_SOURCE_FILES:
        download_fitacfs_from_globus(fitDir, startDate, 'fitacf_30')

    convert_fitacf_to_netcdf(startDate, endDate, fitDir, netDir, 3.0)
    remove_converted_files(fitDir)

    os.rmdir(fitDir)

    totalTime = helper.getTimeString(time.time() - startTime)
    emailSubject = '"RawACF Download and Conversion Complete"'
    emailBody = '"Finished downloading and converting {month} fitACF data\nTotal time: {time}"'.format(
        month=startDate.strftime('%Y/%m'), time=totalTime)
    helper.send_email(emailSubject, emailBody)
Example #2
0
def user_register(data, stage, app_url):
    team_id = data['ctf_time_team_id']
    team_id = None if team_id == '' else int(team_id)
    email = data['email']
    password = data['password']
    team_name = data['team_name']

    with psql_connection(SECRETS['DB_PASSWORD']) as psql:
        with psql.cursor() as cursor:
            LOGGER.info('USER REGISTER {}'.format(email))
            try:
                cursor.execute(
                    'INSERT INTO users VALUES (DEFAULT, now(), '
                    'NULL, %s, crypt(%s, gen_salt(\'bf\', 10)), '
                    '%s, %s)', (email, password, team_name, team_id))
            except psycopg2.IntegrityError as exception:
                if 'email' in exception.diag.constraint_name:
                    return api_response(409, 'duplicate email')
                return api_response(409, 'duplicate team name')
            cursor.execute('SELECT id FROM users where email=%s;', (email, ))
            user_id = cursor.fetchone()[0]
            confirmation_id = str(uuid.uuid4())
            cursor.execute('INSERT INTO confirmations VALUES (%s, %s);',
                           (confirmation_id, user_id))
        psql.commit()

    confirmation_url = '{}/#/confirm/{}'.format(app_url, confirmation_id)
    body = 'Please confirm your account creation:\n\n{}\n'.format(
        confirmation_url)
    send_email('OOO Account Registration <*****@*****.**>',
               email,
               '[OOO] Please Confirm Your Registration',
               body,
               stage=stage)
    return api_response(201)
def user_confirm(data, stage):
    confirmation_id = data["id"]
    with psql_connection(DB_PASSWORD, "scoreboard") as psql:
        with psql.cursor() as cursor:
            LOGGER.info("CONFIRM: {}".format(confirmation_id))
            cursor.execute("SELECT user_id FROM confirmations where id=%s;",
                           (confirmation_id, ))
            response = cursor.fetchone()
            if not response:
                return api_response(
                    409,
                    "invalid confirmation or confirmation already completed")
            user_id = response[0]
            cursor.execute("DELETE FROM confirmations where id=%s;",
                           (confirmation_id, ))
            cursor.execute(
                "UPDATE users SET date_confirmed=now() where id=%s;",
                (user_id, ))
            cursor.execute("SELECT email FROM users where id=%s;", (user_id, ))
            email = cursor.fetchone()[0]
        psql.commit()

        LOGGER.info("EMAIL: {}".format(email))
    body = ("Your registration to DEF CON CTF Quals is complete.\n\n"
            "https://scoreboard.oooverflow.io/\n")
    send_email(
        "OOO Account Registration <*****@*****.**>",
        email,
        "[OOO] Registration Complete",
        body,
        stage=stage,
    )
    return api_response(200, "confirmation complete")
Example #4
0
def failedToGrabData(year):
    # Send an email and end the script if rsync didn't succeed
    emailSubject = '"Unsuccessful attempt to grab {0} Globus  Data"'.format(
        year)
    emailBody = '"Tried to copy {0} from Globus {1} times, but did not succeed. \nSee logfile for more details."'.format(
        year, MAX_NUM_TRIES)
    helper.send_email(emailSubject, emailBody)
def modify_user():
    password = request.form.get("password")
    passwordHash = generate_password_hash(password)
    if session['nivel2'] != '4':
        if request.form.get("nivel") == '4':
            mail_body = "This is message from"
            mail_html = """<html>
            <p style='margin: 0px 0px 8px; text-overflow: ellipsis; overflow-wrap: break-word; color: rgb(23, 43, 77); font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Noto Sans", Ubuntu, "Droid Sans", "Helvetica Neue", sans-serif; font-size: 14px; font-style: normal; font-variant-ligatures: normal; font-variant-caps: normal; font-weight: 400; letter-spacing: normal; orphans: 2; text-align: start; text-indent: 0px; text-transform: none; white-space: normal; widows: 2; word-spacing: 0px; -webkit-text-stroke-width: 0px; background-color: rgb(255, 255, 255); text-decoration-thickness: initial; text-decoration-style: initial; text-decoration-color: initial;'>Desde la Secretar&iacute;a de Modernizaci&oacute;n comunicamos que fue habilitado para el segundo nivel de la capacitaci&oacute;n en la Plataforma GDE Training.</p>
            <p style='margin: 0px 0px 8px; text-overflow: ellipsis; overflow-wrap: break-word; color: rgb(23, 43, 77); font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Noto Sans", Ubuntu, "Droid Sans", "Helvetica Neue", sans-serif; font-size: 14px; font-style: normal; font-variant-ligatures: normal; font-variant-caps: normal; font-weight: 400; letter-spacing: normal; orphans: 2; text-align: start; text-indent: 0px; text-transform: none; white-space: normal; widows: 2; word-spacing: 0px; -webkit-text-stroke-width: 0px; background-color: rgb(255, 255, 255); text-decoration-thickness: initial; text-decoration-style: initial; text-decoration-color: initial;'>Para ingresar al segundo nivel de la capacitacion ingrese a la plataforma <a href="www.gdetraining.com">https://www.gdetraining.com</a> con sus credenciales</p>
            <p style='margin: 0px; text-overflow: ellipsis; overflow-wrap: break-word; color: rgb(23, 43, 77); font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Noto Sans", Ubuntu, "Droid Sans", "Helvetica Neue", sans-serif; font-size: 14px; font-style: normal; font-variant-ligatures: normal; font-variant-caps: normal; font-weight: 400; letter-spacing: normal; orphans: 2; text-align: start; text-indent: 0px; text-transform: none; white-space: normal; widows: 2; word-spacing: 0px; -webkit-text-stroke-width: 0px; background-color: rgb(255, 255, 255); text-decoration-thickness: initial; text-decoration-style: initial; text-decoration-color: initial;'>Canales de comunicaci&oacute;n para acceder a tutores<br>What app 388 4383563 7-13h d&iacute;as laborables<br>Mail&nbsp;<a href="mailto:[email protected]" style="background-color: transparent; color: rgb(23, 43, 77);">[email protected]</a></p>
            </html>"""
            mail_subject = "Segundo nivel GDE Training"
            mail_recipient = request.form.get("email")
            send_email(mail_body, mail_html, mail_subject, mail_recipient)
    user_Id = request.form.get('id')
    cuil = request.form.get('cuil')
    username = request.form.get('username')
    password = passwordHash
    reparticion = request.form.get('reparticion')
    email = request.form.get('email')
    nombre = request.form.get('nombre')
    apellido = request.form.get('apellido')
    nivel = request.form.get("nivel")
    modify_user_query(user_Id, cuil, username, password, reparticion, email,
                      nombre, apellido, nivel)
    return redirect(url_for("usersApp.all_users"))
Example #6
0
def user_confirm(data, stage):
    confirmation_id = data['id']
    with psql_connection(SECRETS['DB_PASSWORD']) as psql:
        with psql.cursor() as cursor:
            LOGGER.info('CONFIRM: {}'.format(confirmation_id))
            cursor.execute('SELECT user_id FROM confirmations where id=%s;',
                           (confirmation_id, ))
            response = cursor.fetchone()
            if not response:
                return api_response(
                    409, 'invalid confirmation or confirmation'
                    ' already completed')
            user_id = response[0]
            cursor.execute('DELETE FROM confirmations where id=%s;',
                           (confirmation_id, ))
            cursor.execute(
                'UPDATE users SET date_confirmed=now() '
                'where id=%s;', (user_id, ))
            cursor.execute('SELECT email FROM users where id=%s;', (user_id, ))
            email = cursor.fetchone()[0]
        psql.commit()

        LOGGER.info('EMAIL: {}'.format(email))
    body = ('Your registration to DEF CON 2018 CTF Quals is complete.\n\n'
            'https://scoreboard.oooverflow.io/\n')
    send_email('OOO Account Registration <*****@*****.**>',
               email,
               '[OOO] Registration Complete',
               body,
               stage=stage)
    return api_response(200, 'confirmation complete')
def convert_rawacf_to_fitacf_and_netcdf(startDate, endDate, rawDir, fitDir, netDir):

    raw_to_nc.main(startDate, endDate, rawDir,fitDir,netDir)
    dateString = startDate.strftime('%Y/%m')

    emailSubject = '"{date} rawACF to netCDF Conversion Successful"'.format(date = dateString)
    emailBody    = 'Finished converting {date} rawACF files to fitACF and netCDF'.format(date = dateString)
    helper.send_email(emailSubject, emailBody)
Example #8
0
    def get_bug_report(self):
        # set gzip header
        cherrypy.response.headers['Content-Type'] = 'application/gzip'
        # create return tuple
        return_tuple = {}
        return_tuple['status'] = "failed"
        return_tuple['warning'] = ""
        return_tuple['error'] = ""

        # parse json encoded input
        input = helper.convert_dict_values_to_utf8(cherrypy.request.json)

        # user language
        language = ""
        if input.has_key("language") == True:
            language = input['language']
        # if the user sends a language, which is not german, take the default language setting
        if language != "de":
            language = Config().get_param("default_language")
        # initialize the translator object with the user's choosen language
        translator = Translator(language)

        # bug report variables
        if input.has_key("file_name") == False:
            return_tuple['error'] = translator.translate(
                "message", "no_bug_report_file_name")
            return helper.zip_data(return_tuple)
        if input.has_key("bug_report") == False:
            return_tuple['error'] = translator.translate(
                "message", "no_bug_report_contents")
            return helper.zip_data(return_tuple)

        # save bug report
        try:
            bug_report_folder = os.path.join(Config().get_param("logs_folder"),
                                             "client")
            if os.path.exists(bug_report_folder) == False:
                os.makedirs(bug_report_folder)
            file = open(
                os.path.join(bug_report_folder,
                             input['file_name'].split("/")[-1]), 'w')
            file.write(input['bug_report'])
            file.close()
        except IOError as e:
            pass

        # send mail to the admin
        helper.send_email(
            "OSMRouter: New bug report", "%s\n\n%s" %
            (input['file_name'].split("/")[-1], input['bug_report']))

        # convert return_tuple to json and zip it, before returning
        return_tuple['status'] = "ok"
        return helper.zip_data(return_tuple)
def register_user():
    if request.method == "GET":
        return render_template("authentication/register.html")
    elif request.method == "POST":
        nivel = 1
        if not request.form.get('cuil'):
            return error("Revise el cuil e intente nuevamente")
        if not request.form.get('username'):
            return error("Revise el username e intente nuevamente")
        if not request.form.get('reparticion'):
            return error("Revise la reparticion e intente nuevamente")
        if not request.form.get('email'):
            return error("Revise el email e intente nuevamente")
        if not request.form.get('nombre'):
            return error("Revise el nombre e intente nuevamente")
        if not request.form.get('apellido'):
            return error("Revise el apellido e intente nuevamente")
        try:
            if session["rol"] != 3:
                rol = 1
            else:
                rol = request.form.get('rol')
            cuil = request.form.get('cuil'),
            username = request.form.get('username'),
            password = generate_password_hash(request.form.get('password')),
            reparticion = request.form.get('reparticion'),
            email = request.form.get('email'),
            nombre = request.form.get('nombre'),
            apellido = request.form.get('apellido'),
            rol = rol,
            nivel = nivel
            register_query(cuil, username, password, reparticion, email,
                           nombre, apellido, rol, nivel)
            body = "This is message from" + request.form.get("email") + "\r\n"
            html = """<html>
            <p style='margin: 0px 0px 8px; text-overflow: ellipsis; overflow-wrap: break-word; color: rgb(23, 43, 77); font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Noto Sans", Ubuntu, "Droid Sans", "Helvetica Neue", sans-serif; font-size: 14px; font-style: normal; font-variant-ligatures: normal; font-variant-caps: normal; font-weight: 400; letter-spacing: normal; orphans: 2; text-align: start; text-indent: 0px; text-transform: none; white-space: normal; widows: 2; word-spacing: 0px; -webkit-text-stroke-width: 0px; background-color: rgb(255, 255, 255); text-decoration-thickness: initial; text-decoration-style: initial; text-decoration-color: initial;'>Desde la Secretar&iacute;a de Modernizaci&oacute;n comunicamos que fue habilitado para la capacitaci&oacute;n en la Plataforma GDE Training.</p>
            <p style='margin: 0px 0px 8px; text-overflow: ellipsis; overflow-wrap: break-word; color: rgb(23, 43, 77); font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Noto Sans", Ubuntu, "Droid Sans", "Helvetica Neue", sans-serif; font-size: 14px; font-style: normal; font-variant-ligatures: normal; font-variant-caps: normal; font-weight: 400; letter-spacing: normal; orphans: 2; text-align: start; text-indent: 0px; text-transform: none; white-space: normal; widows: 2; word-spacing: 0px; -webkit-text-stroke-width: 0px; background-color: rgb(255, 255, 255); text-decoration-thickness: initial; text-decoration-style: initial; text-decoration-color: initial;'>Para ingreso&nbsp;<a href="https://www.gdetraining.com/" rel="noreferrer nofollow" style="background-color: transparent; color: rgb(23, 43, 77);" target="_blank">https://www.gdetraining.com/</a><br>Usuario:""" + request.form.get(
                'username') + """<br>Clave:""" + request.form.get(
                    'password') + """</p>
            <p style='margin: 0px 0px 8px; text-overflow: ellipsis; overflow-wrap: break-word; color: rgb(23, 43, 77); font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Noto Sans", Ubuntu, "Droid Sans", "Helvetica Neue", sans-serif; font-size: 14px; font-style: normal; font-variant-ligatures: normal; font-variant-caps: normal; font-weight: 400; letter-spacing: normal; orphans: 2; text-align: start; text-indent: 0px; text-transform: none; white-space: normal; widows: 2; word-spacing: 0px; -webkit-text-stroke-width: 0px; background-color: rgb(255, 255, 255); text-decoration-thickness: initial; text-decoration-style: initial; text-decoration-color: initial;'>La capacitaci&oacute;n finaliza el 25 de febrero del corriente a&ntilde;o, espacio en el cu&aacute;l deber&aacute; completar todas las actividades requeridas.</p>
            <p style='margin: 0px; text-overflow: ellipsis; overflow-wrap: break-word; color: rgb(23, 43, 77); font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Noto Sans", Ubuntu, "Droid Sans", "Helvetica Neue", sans-serif; font-size: 14px; font-style: normal; font-variant-ligatures: normal; font-variant-caps: normal; font-weight: 400; letter-spacing: normal; orphans: 2; text-align: start; text-indent: 0px; text-transform: none; white-space: normal; widows: 2; word-spacing: 0px; -webkit-text-stroke-width: 0px; background-color: rgb(255, 255, 255); text-decoration-thickness: initial; text-decoration-style: initial; text-decoration-color: initial;'>Canales de comunicaci&oacute;n para acceder a tutores<br>What app 388 4383563 7-13h d&iacute;as laborables<br>Mail&nbsp;<a href="mailto:[email protected]" style="background-color: transparent; color: rgb(23, 43, 77);">[email protected]</a></p>
            </html>"""
            subject = 'Confirmacion de registro Secretaria de modernizacion'
            recipient = request.form.get("email")
            try:
                send_email(body, html, subject, recipient)
            except:
                error(
                    "Hubo un error al enviar el mail. Enviarlo manualmente o revisar el mail"
                )
            return redirect(url_for("cursos"))
        except:
            return error("Revise los campos e intente nuevamente")
Example #10
0
def save_exchange(buyer, tstamp, fee, data):
    """ Save a new SuiExchange entity if not saved yet, key_name is method_order_number.
        @param data : {'item_id':'sup1','quantity':1,'price':10,'order_number':'2343-3422','currency':'GBP','method':'GC'}
        @exception : db error
    """
    m = data['method']
    on = data['order_number']
    keyname = '%s_%s' % (m, on)
    exe = SuiExchange.get_by_key_name(keyname)
    if isinstance(buyer, basestring):
        ukey = buyer
        buyer = helper.query_user_by_key(ukey)
    else:
        ukey = buyer.key().name()
    if exe is None:
        exe = SuiExchange(key_name=keyname, user=ukey, xtime=tstamp)
        q = int(data['quantity'])
        pkg = data['item_id']
        exe.points = q * gPackages[pkg.lower()]['pts']
        exe.quantity = q
        exe.price = float(data['price'])
        exe.orderef = '%s' % on
        exe.currency = data['currency']
        exe.method = data['method']
        exe.buyerid = '%s' % data['buyer']
        exe.package = pkg
        exe.fee = 1.0 * (fee or 1)
        taskqueue.add(url='/task/log',
                      params={
                          'usr':
                          ukey,
                          'act':
                          'bought %s,%s+%s' % (pkg, buyer.points, exe.points),
                          'dt':
                          str(datetime.utcnow())
                      })
        try:
            exe.put()
            buyer.points += exe.points
            buyer.save()
            logging.debug('SuiExchange saved, %s, %s' % (m, on))
            taskqueue.add(url='/task/dau',
                          params={
                              'usr': buyer.key().name(),
                              'act': 'pay',
                              'par': pkg,
                              'qty': str(q)
                          })
        except Exception, e:
            logging.exception('SuiExchange.put() failed:%s' % e)
            helper.send_email('Suinova test payment failure notification',
                              str(e))
            raise
def download_files_from_globus(rawDir, netDir, date):
    # Start Globus Connect Personal and establish connection
    # Also allow access to /project/superdarn/data/
    subprocess.call('{0} -start -restrict-paths \'rw~/,rw/project/superdarn/data\' &'.format(helper.GLOBUS_PATH), shell=True)

    # Initiate the transfer from Globus to APL
    subprocess.call('nohup /project/superdarn/software/python-3.8.1/bin/python3 /homes/superdarn/globus/sync_radar_data_globus.py -y {0} -m {1} {2}'.format(date.year, date.month, rawDir), shell=True)

    # Stop Globus Connect Personal
    subprocess.call('{0} -stop'.format(helper.GLOBUS_PATH), shell=True)

    emailSubject = '"{0} rawACF Data Successfully Downloaded"'.format(date.strftime('%Y/%m'))
    emailBody    = '"{0} rawACF source files have been downloaded. Starting conversion to fitACF and netCDF."'.format(date.strftime('%Y/%m'))
    helper.send_email(emailSubject, emailBody)
Example #12
0
def profile(web,args=None):
    """ Readers apply for authorship or authors update their profile.
        Form is submitted and it returns the main page with authors as pageview.
        User input will be saved in SuiAuthor entity but SuiUser.role is still 'R'.
        An email will be sent to admin who will then review the author and change R to A, and notify the author.
    """
    if not web.logged_in:
        web.redirect_with_msg('Login required')
        return
    me = web.user
    parms=['title','fname','lname','email','address','job','intro','web','blog','facebook','twitter','linkedin','other','bda']
    title,fname,lname,email,address,job,intro,webs,blog,facebook,twitter,linkedin,other,bda=web.get_params(parms)
    if me.isAuthor():
        a = SuiAuthor.get_by_key_name(me.key().name())
    else:
        a = SuiAuthor(key_name=me.key().name(),name=me.name)
    if title: a.title = title
    if fname: a.firstname = fname
    if lname: a.lastname = lname
    if email: a.email = email
    if address: a.address = address
    if job: a.job = job
    if intro:
        ins = intro.replace('"','&quot;').replace('<','&lt;')
        a.intro = ins 
        #a.intro = ''.join(['<p>%s</p>'%ls.strip() for ls in intro.replace('<','&lt;').split('\n')])
    lnks = []
    if webs: lnks.append('Web: %s'%webs)
    if blog: lnks.append('Blog: %s'%blog)
#    if facebook: lnks.append('Facebook: %s'%facebook)
    if facebook: a.fbpage = facebook
    if twitter: lnks.append('Twitter: %s'%twitter)
    if linkedin: lnks.append('LinkedIn: %s'%linkedin)
    if other: lnks.append('Other: %s'%other)
    if lnks:
        a.links = '[%s]'%','.join(['"%s"'%s for s in lnks])  #["web:http:..","twitter:",..]
    try:
        a.put()
        if me.role == 'R':
            me.role = 'a'
            me.save()
            msg = APP_MSG % (me.name,me.email or '',facebook,'\r\n'.join(lnks),me.key().name(),bda)
            helper.send_email('[Suinova Comics] %s appled for Authorship'%me.name,msg,'*****@*****.**')
        else:
            me.detach_author()
    except Exception,e:
        logging.exception(e)
        web.redirect_with_msg('Server error, retry later')
        return
def main(date):

    startTime = time.time()
    emailSubject = '"Starting Zenodo Upload"'
    emailBody = '"Starting to upload {0} netCDF files to Zenodo"'.format(
        date.strftime('%Y-%m'))
    helper.send_email(emailSubject, emailBody)

    upload_to_zenodo(SANDBOX, date)

    totalTime = helper.getTimeString(time.time() - startTime)
    emailSubject = '"Finished Zenodo Upload"'
    emailBody = '"Finished uploading {0} netCDF files\nTotal time: {1}"'.format(
        date.strftime('%Y-%m'), totalTime)
    helper.send_email(emailSubject, emailBody)
    def get_bug_report(self):
        # set gzip header
        cherrypy.response.headers['Content-Type'] = 'application/gzip'
        # create return tuple
        return_tuple = {}
        return_tuple['status'] = "failed"
        return_tuple['warning'] = ""
        return_tuple['error'] = ""

        # parse json encoded input
        input = helper.convert_dict_values_to_utf8( cherrypy.request.json )

        # user language
        language = ""
        if input.has_key("language") == True:
            language = input['language']
        # if the user sends a language, which is not german, take the default language setting
        if language != "de":
            language = Config().get_param("default_language")
        # initialize the translator object with the user's choosen language
        translator = Translator(language)

        # bug report variables
        if input.has_key("file_name") == False:
            return_tuple['error'] = translator.translate("message", "no_bug_report_file_name")
            return helper.zip_data(return_tuple)
        if input.has_key("bug_report") == False:
            return_tuple['error'] = translator.translate("message", "no_bug_report_contents")
            return helper.zip_data(return_tuple)

        # save bug report
        try:
            bug_report_folder = os.path.join( Config().get_param("logs_folder"), "client")
            if os.path.exists(bug_report_folder) == False:
                os.makedirs(bug_report_folder)
            file = open(os.path.join(bug_report_folder, input['file_name'].split("/")[-1]), 'w')
            file.write(input['bug_report'])
            file.close()
        except IOError as e:
            pass

        # send mail to the admin
        helper.send_email("OSMRouter: New bug report", "%s\n\n%s"
                % (input['file_name'].split("/")[-1], input['bug_report']) )

        # convert return_tuple to json and zip it, before returning
        return_tuple['status'] = "ok"
        return helper.zip_data(return_tuple)
Example #15
0
def feedback(web,args=None):
    """ Forward a message to admin.
    """
    if not web.logged_in:
        web.redirect_with_msg('Login required.')
        return
    msg = web.get_param('msg')
    if not msg:
        web.redirect_with_msg('Empty message.')
        return
    sender = web.user
    msg2 = 'Sender: %s (%s), email: %s, on %s (GMT)\n\n%s' % (sender.name,sender.key().name(),sender.email,datetime.utcnow(),msg)
    import helper
    helper.send_email('User Message via Contact Page',msg2)
    web.redirect('/')
    
Example #16
0
def auth_password_request(email):
    '''This program requests a password change and sends the user a reset code'''
    data = get_data()
    valid_email = -1
    for user in data['users']:
        if user['email'] == email:
            valid_email = 1
            break
    if valid_email == -1:
        raise InputError("Invalid Email")
    #Create a temporary field for this user called reset_code
    #When entered correctly it will be reset
    user['reset_code'] = generate_reset_code()
    # Format of this message is important for else the subject field won't
    # actually be the email subject.
    send_email(email, str(user['reset_code']))
    return {}
Example #17
0
def main():
    _configure_logger('email_annonce.log')
    logger.info("##### Le Pere Noel de la Fremo #####")

    with open(__SECURE_DATA_FILEPATH, 'r') as stream:
        secure_data = yaml.safe_load(stream)

    # debug
    #secure_data['personnes'] = {'nboutin' : {'email':'*****@*****.**'}}
    # debug

    with open(__EMAIL_TEXT_FILE_PATH, 'r') as file:
        body = file.read()

    for id_, id_param in secure_data['personnes'].items():
        helper.send_email(secure_data['sender_email'], secure_data['sender_pwd'], __EMAIL_OBJECT, body, id_param['email'])

    logger.info("Finish")
def main():
    date = START_DATE
    while date >= END_DATE:
        emailSubject = '"PROCESS ALL STARTING"'
        emailBody = 'Starting to download and process {0} data'.format(
            date.strftime('%Y%m'))
        helper.send_email(emailSubject, emailBody)

        download_and_process_fitacfs.main(date)
        #download_and_process_rawacfs.main(date)
        upload_nc_to_zenodo.main(date)

        emailSubject = '"PROCESS ALL COMPLETE"'
        emailBody = 'Finished downloading and processing {0} data'.format(
            date.strftime('%Y%m'))
        helper.send_email(emailSubject, emailBody)

        date -= relativedelta(months=1)
Example #19
0
def accept_author(web, args=None):
    """ After reviewing author application, click Accept link and here.
        Change SuiUser.role from 'R' to 'A', send this user an email as notification.
    """
    web.require_admin()
    if not args:
        logging.warning('admin/accept_author/None')
        web.fail('Invalid author accept call')
        return
    uid = args[0]
    bda = web.request.get('bda')
    a = SuiAuthor.get_by_key_name(uid)
    if not a:
        web.fail('User not recorded')
        return
    u = SuiUser.get_by_key_name(uid)
    if u.role != 'A':
        u.role = 'A'
        a.confirmed = True
        try:
            db.put([a, u])
            u.recache()
            a.recache()
            if uid.startswith('fb_'):
                lts = WELCOME_LETTER % 'http://apps.facebook.com/suicomics/'
            else:
                lts = WELCOME_LETTER % 'http://suicomics.appspot.com/'
            if bda == '1':
                lts += 'gift'
            helper.send_email('Application Accepted', lts, a.email)
            if bda == '1':
                bu = SuiBirthdayUser.get_by_key_name(uid)
                if not bu:
                    bu = SuiBirthdayUser(
                        key_name=uid, name=u.name, access_token=u.access_token
                    )  #update access_token when user change password
                    bu.creator = True
                bu.put()
            web.succeed('Done')
        except Exception, e:
            logging.exception(e)
            web.fail('Accept not successful, please notify admin')
            return
Example #20
0
def send_all_email(data, secure_data):

    subject = 'Le Père Noël de la Frémo: Tirage au sort'

    for id_, id_param in data.items():
        toaddr = secure_data['personnes'][id_]['email']
        # DEBUG
        # toaddr = secure_data['receiver_test']
        # DEBUG

        to_gift_id = id_param['history'][-1]
        to_fullname = data[to_gift_id]['fullname']

        with open(__BODY, 'r') as file:
            body = file.read().format(id_param['fullname'], to_fullname)

        logger.debug("{} -> {} ({})".format(id_, to_fullname, to_gift_id))
        helper.send_email(secure_data['sender_email'],
                          secure_data['sender_pwd'], subject, body, toaddr)
Example #21
0
def getBasFileList():
    # Make sure the BAS server is reachable
    if not BASServerConnected():
        # Send email if BAS couldn't be reached  
        emailSubject = '"Unable to reach BAS"'
        emailBody    = 'Unable to reach BAS after trying for {hours} hours.'.format(hours = RETRY * DELAY / 3600)
        helper.send_email(emailSubject, emailBody)
        sys.exit('{message}'.format(message = emailBody))

    os.makedirs(BAS_FILE_LIST_DIR, exist_ok=True)

    # TODO: Get BAS DAT files and Globus raw and dat files
    # globus login
    # globus ls 'c02cb494-1515-11e9-9f9f-0a06afd4a22e:/chroot/sddata/dat/2006'
    # globus ls 'c02cb494-1515-11e9-9f9f-0a06afd4a22e:/chroot/sddata/raw/2005'

    year = BAS_START_DATE.year
    endYear = BAS_END_DATE.year
    while year <= endYear:
        print('\nGetting BAS data for {0}\n'.format(year))
        filename = '{dir}/{yr}_basRawFiles.txt'.format(dir = BAS_FILE_LIST_DIR, yr = year)

        # Get a list of all rawACF files on BAS for the given year and put them in a file
        os.system('ssh apl@{bas} ls -R /sddata/raw/{yr}/ > {fname}'.format(bas = helper.BAS_SERVER, yr = year, fname = filename))
        # Go through the file line by line and add each line to the appropriate daily text file
        with open(filename) as mainFileList:
            lines = mainFileList.readlines()        
            for line in lines:
                rawFilename = line.strip()
                extension = rawFilename.split('.')[-1]
                if not extension == 'bz2':
                    # This line isn't a rawACF filename
                    continue
                
                day = rawFilename.split('.')[0]
                print('Updating {0} file'.format(day))
                radar = rawFilename.split('.')[3]
                radarFileList = '{dir}/{d}_{r}.txt'.format(dir = BAS_FILE_LIST_DIR, d = day, r = radar)
                with open(radarFileList, "a+") as fp: 
                    fp.write(rawFilename + '\n')

        year += 1
def user_register(data, stage, app_url):
    team_id = data["ctf_time_team_id"]
    team_id = None if team_id == "" else int(team_id)
    email = data["email"]
    password = data["password"]
    team_name = data["team_name"]

    with psql_connection(DB_PASSWORD, "scoreboard") as psql:
        with psql.cursor() as cursor:
            LOGGER.info("USER REGISTER {}".format(email))
            try:
                cursor.execute(
                    "INSERT INTO users (id, date_created, date_updated, ctf_time_team_id, "
                    "email, password, team_name) VALUES (DEFAULT, now(), now(), %s, %s, "
                    "crypt(%s, gen_salt('bf', 10)), %s)",
                    (team_id, email, password, team_name),
                )
            except psycopg2.IntegrityError as exception:
                if "email" in exception.diag.constraint_name:
                    return api_response(409, "duplicate email")
                return api_response(409, "duplicate team name")
            cursor.execute("SELECT id FROM users where email=%s;", (email, ))
            user_id = cursor.fetchone()[0]
            confirmation_id = str(uuid.uuid4())
            cursor.execute(
                "INSERT INTO confirmations (id, user_id) VALUES (%s, %s);",
                (confirmation_id, user_id),
            )
        psql.commit()

    confirmation_url = "{}/#/confirm/{}".format(app_url, confirmation_id)
    body = "Please confirm your account creation:\n\n{}\n".format(
        confirmation_url)
    send_email(
        "OOO Account Registration <*****@*****.**>",
        email,
        "[OOO] Please Confirm Your Registration",
        body,
        stage=stage,
    )
    return api_response(201)
Example #23
0
def accept_author(web,args=None):
    """ After reviewing author application, click Accept link and here.
        Change SuiUser.role from 'R' to 'A', send this user an email as notification.
    """
    web.require_admin()
    if not args:
        logging.warning('admin/accept_author/None')
        web.fail('Invalid author accept call')
        return
    uid = args[0]
    bda = web.request.get('bda')
    a = SuiAuthor.get_by_key_name(uid)
    if not a:
        web.fail('User not recorded')
        return
    u = SuiUser.get_by_key_name(uid)
    if u.role != 'A':
        u.role = 'A'
        a.confirmed = True
        try:
            db.put([a,u])
            u.recache()
            a.recache()
            if uid.startswith('fb_'):
                lts = WELCOME_LETTER % 'http://apps.facebook.com/suicomics/'
            else:
                lts = WELCOME_LETTER % 'http://suicomics.appspot.com/'
            if bda == '1':
                lts += 'gift'
            helper.send_email('Application Accepted',lts,a.email)
            if bda == '1':
                bu = SuiBirthdayUser.get_by_key_name(uid)
                if not bu:
                    bu = SuiBirthdayUser(key_name=uid,name=u.name,access_token=u.access_token) #update access_token when user change password
                    bu.creator = True
                bu.put()
            web.succeed('Done')
        except Exception,e:
            logging.exception(e)
            web.fail('Accept not successful, please notify admin')
            return
Example #24
0
def main():
    startTime = time.time()
    emailSubject = '"Starting Data Check"'
    emailBody    = 'Starting BAS vs APL data check'
    helper.send_email(emailSubject, emailBody)
    
    getBasFileList()
    radarList = helper.get_radar_list()

    # TODO: Add DAT file check

    date = BAS_START_DATE
    data = {}
    while date <= BAS_END_DATE:
        day = date.strftime('%Y%m%d')
        data[day] = []
        print('Comparing data between BAS and APL on {d}'.format(d = day))
        for radar in radarList:
            basDataExists = bas_data(date, radar)
            aplDataExists = apl_data(date, radar)
            result = get_result(basDataExists, aplDataExists)        
 
            data[day].append({
                'radar': radar,
                'result': result
            })
                        
        date += dt.timedelta(days=1)

    outputFile = '{dir}/data_status_{date}.txt'.format(dir = DATA_STATUS_DIR, date = BAS_END_DATE.strftime('%Y%m%d'))
    with open(outputFile, 'w') as outfile:
        json.dump(data, outfile)
    
    totalTime = helper.getTimeString(time.time() - startTime)
    emailSubject = '"Data Status Check Complete"'
    emailBody    = '"Finished checking BAS data vs APL data.\nTotal check runtime: {0}\nNew JSON file created: {1}"'.format(totalTime, outputFile)
    helper.send_email(emailSubject, emailBody)

    if REMOVE_BAS_FILE_LIST:
        os.system('rm -rf {bas_dir}'.format(bas_dir = BAS_FILE_LIST_DIR))
Example #25
0
def main():
    startTime = time.time()
    emailSubject = '"Starting Data Check"'
    emailBody = 'Starting Globus vs Zenodo data check'
    helper.send_email(emailSubject, emailBody)

    getGlobusFileList()
    getZenodoFileList()

    radarList = helper.get_radar_list()

    date = START_DATE
    data = {}
    while date <= END_DATE:
        day = date.strftime('%Y%m%d')
        data[day] = []
        print('{0} - Comparing data between Globus and Zenodo on {1}'.format(
            time.strftime('%Y-%m-%d %H:%M'), day))
        for radar in radarList:
            remoteDataExists = getRemoteData(date, radar)
            zenodoDataExists = getZenodoData(date, radar)
            result = get_result(remoteDataExists, zenodoDataExists)

            data[day].append({'radar': radar, 'result': result})

        date += dt.timedelta(days=1)

    outputFile = '{0}/{1}_data_status.json'.format(DATA_STATUS_DIR,
                                                   END_DATE.strftime('%Y%m%d'))
    with open(outputFile, 'w') as outfile:
        json.dump(data, outfile)

    totalTime = helper.getTimeString(time.time() - startTime)
    emailSubject = '"Data Status Check Complete"'
    emailBody = '"Finished checking Globus vs Zenodo data.\nTotal check runtime: {0}\nNew JSON file created: {1}"'.format(
        totalTime, outputFile)
    helper.send_email(emailSubject, emailBody)

    if REMOVE_REMOTE_FILE_LIST:
        os.system('rm -rf {0}'.format(GLOBUS_FILE_LIST_DIR))
Example #26
0
def download_fitacfs_from_globus(fitDir, date, pattern):
    # Start Globus Connect Personal and establish connection
    # Also allow access to /project/superdarn/data/
    subprocess.call(
        '{0} -start -restrict-paths \'rw~/,rw/project/superdarn/data/fitacf\' &'
        .format(helper.GLOBUS_PATH),
        shell=True)

    # Initiate the Globus -> APL transfer
    subprocess.call(
        'nohup /project/superdarn/software/python-3.8.1/bin/python3 /homes/superdarn/globus/sync_radar_data_globus.py -y {0} -m {1} -t {2} {3}'
        .format(date.year, date.month, pattern, fitDir),
        shell=True)

    # Stop Globus Connect Personal
    #subprocess.call('{0} -stop'.format(helper.GLOBUS_PATH), shell=True)

    emailSubject = '"{0} {1} Data Successfully Downloaded"'.format(
        date.strftime('%Y/%m'), pattern)
    emailBody = '"{0} {1} source files have been downloaded. Starting conversion to netCDF."'.format(
        date.strftime('%Y/%m'), pattern)
    helper.send_email(emailSubject, emailBody)
Example #27
0
def save_exchange(buyer,tstamp,fee,data):
    """ Save a new SuiExchange entity if not saved yet, key_name is method_order_number.
        @param data : {'item_id':'sup1','quantity':1,'price':10,'order_number':'2343-3422','currency':'GBP','method':'GC'}
        @exception : db error
    """
    m = data['method']
    on = data['order_number']
    keyname = '%s_%s' % (m, on)
    exe = SuiExchange.get_by_key_name(keyname)
    if isinstance(buyer,basestring):
        ukey = buyer
        buyer = helper.query_user_by_key(ukey)
    else:
        ukey = buyer.key().name() 
    if exe is None:
        exe = SuiExchange(key_name=keyname,user=ukey,xtime=tstamp)
        q = int(data['quantity'])
        pkg = data['item_id']
        exe.points = q * gPackages[pkg.lower()]['pts']
        exe.quantity = q
        exe.price = float(data['price'])
        exe.orderef = '%s'%on
        exe.currency = data['currency']
        exe.method = data['method']
        exe.buyerid = '%s'%data['buyer']
        exe.package = pkg
        exe.fee = 1.0 * (fee or 1)
        taskqueue.add(url='/task/log',params={'usr':ukey,'act':'bought %s,%s+%s'%(pkg,buyer.points,exe.points),'dt':str(datetime.utcnow())})
        try:
            exe.put()
            buyer.points += exe.points
            buyer.save()
            logging.debug('SuiExchange saved, %s, %s' % (m, on))
            taskqueue.add(url='/task/dau',params={'usr':buyer.key().name(),'act':'pay','par':pkg,'qty':str(q)})
        except Exception,e:
            logging.exception('SuiExchange.put() failed:%s'%e)
            helper.send_email('Suinova test payment failure notification',str(e))
            raise
def main(date):
    startTime = time.time()
    startDate, endDate = get_first_and_last_days_of_month(date)

    rawDir = startDate.strftime(RAWACF_DIR_FMT) + '/'
    fitDir = startDate.strftime(FITACF_DIR_FMT)
    netDir = startDate.strftime(NETCDF_DIR_FMT)

    os.makedirs(rawDir, exist_ok=True)    
    os.makedirs(fitDir, exist_ok=True)
    os.makedirs(netDir, exist_ok=True)

    if DOWNLOAD_SOURCE_FILES:
        download_source_files(rawDir, netDir, startDate)

    convert_rawacf_to_fitacf_and_netcdf(startDate, endDate, rawDir, fitDir, netDir)

    remove_converted_files(rawDir, fitDir)

    totalTime = helper.getTimeString(time.time() - startTime)
    emailSubject = '"RawACF Download and Conversion Complete"'
    emailBody    = '"Finished downloading and converting {month} RawACF data\nTotal time: {time}"'.format(month = startDate.strftime('%Y/%m'), time = totalTime)
    helper.send_email(emailSubject, emailBody)
Example #29
0
def main():
    date = START_DATE
    while date >= END_DATE:

        emailSubject = '"Starting to update netCDF gflg descriptions"'
        emailBody = 'Starting to update netCDF gflg descriptions for {0}'.format(
            date.strftime('%Y%m'))
        helper.send_email(emailSubject, emailBody)

        netcdfDir = date.strftime(helper.NETCDF_DIR_FMT)
        fileList = glob.glob(os.path.join(netcdfDir, '*.nc'))
        gflgDescription = 'Ground scatter flag for ACF, 1 - ground scatter, 0 - other scatter'

        for file in fileList:
            fh = nc.Dataset(file, mode='r+')
            fh.variables['gflg'].long_name = gflgDescription
            fh.close()

        emailSubject = '"Finished updated netCDF gflg descriptions"'
        emailBody = 'Finished updated netCDF gflg descriptions for {0}'.format(
            date.strftime('%Y%m'))
        helper.send_email(emailSubject, emailBody)

        date -= relativedelta(months=1)
Example #30
0
def send_email():
    
    validated = validateRequest(request)
    if validated != "validated":
        return validated

    req_data = request.get_json()

    # Get items from the helper
    res_data = helper.send_email(req_data)

    # Return 404 if item not found
    if 'error' in res_data:
        response = Response(json.dumps(res_data), status=400, mimetype='application/json')
        response.headers["Access-Control-Allow-Origin"] = "*"
        return response

    # Return response
    response = Response(json.dumps(res_data), status=200, mimetype='application/json')
    response.headers["Access-Control-Allow-Origin"] = "*"
    return response
Example #31
0
def ppapprove(web, args=None):
    """ Called by PayPal through RETURNURL set in SetExpressCheckout command.
        Currently payment is done directly, a better way is to return a review page for the user to confirm payment and
        then come back to send DoExpressCheckoutPayment. But the current method is dirty quick.
        The L_NAME0 items are not displayed in the sandbox PayPal continue page, not sure whether it's the same on production site.
    """
    token = web.request.get('token')
    buyerid = web.request.get('PayerID')
    pkg = web.request.get('pkg')
    ukey = web.request.get('uid')
    if pkg == '':
        cs = web.request.get('CUSTOM')
        if cs.find(':') > 0:
            pkg, ukey = cs.split(':')
    logging.debug('pay.ppapprove: token=%s,buyerid=%s,pkg=%s,ukey=%s' %
                  (token, buyerid, pkg, ukey))
    pg = gPackages[pkg.lower()]
    request = {
        'USER': PP_USER,
        'PWD': PP_PWD,
        'SIGNATURE': PP_SIG,
        'VERSION': PP_VER
    }
    request['PAYERID'] = buyerid
    request['TOKEN'] = token
    request['PAYMENTACTION'] = 'Sale'
    request['AMT'] = '%0.2f' % float(pg['price'])
    request['CURRENCYCODE'] = 'GBP'
    request['METHOD'] = 'DoExpressCheckoutPayment'
    request['CUSTOM'] = '%s:%s' % (ukey, pkg)  # does this return?
    request['DESC'] = '%s. %s' % (pg['item'], pg['description'])
    request['L_NAME0'] = pg['item']
    request['L_NUMBER0'] = pg['id']
    request['L_DESC0'] = pg['description']
    request['L_AMT0'] = request['AMT']
    request['L_QTY0'] = '1'
    requests = urllib.urlencode(request)
    logging.debug('About to send to PayPal: %s' % requests)
    try:
        result = urlfetch.fetch(
            url=PP_ECURL,
            payload=requests,
            method=urlfetch.POST,
            headers={'Content-type': 'application/x-www-form-urlencoded'})
        if result.content.find('ACK=Success') >= 0:
            data = cgi.parse_qs(result.content)
            #token = data['TOKEN'][-1]
            payment_status = data['PAYMENTSTATUS'][0]
            if payment_status == 'Completed':
                data2 = {
                    'method': 'PP',
                    'quantity': '1',
                    'item_id': pkg,
                    'buyer': buyerid
                }
                data2['order_number'] = data['TRANSACTIONID'][0]
                data2['price'] = data['AMT'][0]
                data2['currency'] = data['CURRENCYCODE'][0]
                if 'SETTLEAMT' in data:
                    logging.debug('settlement: %s' % data['SETTLEAMT'][0])
                fee = float(data['FEEAMT'][0])
                tstamp = datetime_from_timestampz(data['ORDERTIME'][0])
                buyer = helper.get_user_by_key(ukey, False)
                if buyer is None:
                    logging.warning('Buyer %s is not user!' % ukey)
                    helper.send_email(
                        'Suinova test payment failure notification',
                        'Buyer %s not found' % ukey)
                    web.response.out.write('Thank you!')
                    return
                try:
                    save_exchange(buyer, tstamp, fee, data2)
                except Exception:
                    pass
                web.response.out.write(THANKU_PAGE)
            else:
                logging.warning(
                    'ppapprove result from DoExpressCheckoutPayment, payment status is %s'
                    % payment_status)
                logging.info(result.content)
                web.response.out.write('Not completed')
        else:
            logging.error(
                'ppapprove send DoExpressCheckoutPayment returned: %s' %
                result.content)
            web.fail(
                'Error contacting PayPal, try later, <a href="/">Go back</a>')
    except Exception, e:
        logging.exception(e)
        web.fail('Error contacting PayPal, try later, <a href="/">Go back</a>')
Example #32
0
 try:
     result = urlfetch.fetch(url=GC_REQUEST_URLS[GC_WHICH],
                             payload=request_xml,
                             method=urlfetch.POST,
                             headers=GC_HEADER)
 except Exception, e:
     logging.exception(e)
     web.error(500)  #will cause resend to google checkout
 try:
     if result.content.find('<charge-amount-notification') > 0:
         #charged, so save in ds
         data = get_google_notification_data(result.content)
         if 'original_buyer' not in data:
             ers = 'merchange-private-data not available in charge-amount-notification'
             logging.error(ers)
             helper.send_email('Suinova test payment failure notification',
                               ers)
             web.succeed()  #return 200 not to resend to Google Checkout
             return
         buyer_id = data['original_buyer']
         buyer = helper.get_user_by_key(buyer_id,
                                        False)  #'gg_%s' % data['buyer'])
         if buyer is None:
             logging.warning('Buyer %s is not user!' % buyer_id)
             helper.send_email('Suinova test payment failure notification',
                               'Buyer %s not found' % buyer_id)
             web.succeed()  #return 200 not to resend to Google Checkout
             return
         gcfee = extract_text_in_tag(result.content, 'total')
         if len(gcfee) > 0:
             try:
                 fee = float(gcfee[0])
Example #33
0
def scrape_gmail_messages(debug, mailbox_to_scrape, access_oauth_token, access_oauth_token_secret, consumer_key, consumer_secret, app_email_info, error_email_info, admins):
    phone_numbers = []
    try:
        start_datetime = datetime.now()
        
        consumer = oauth.Consumer(consumer_key, consumer_secret)
        token = oauth.Token(access_oauth_token, access_oauth_token_secret)
        client = oauth.Client(consumer, token)
        
        # Get the email address from Google contacts
        resp, xmldoc = client.request('https://www.google.com/m8/feeds/contacts/default/full?max-results=0')
        email = get_id(xmldoc)
        
        # Connect with IMAP
        url = 'https://mail.google.com/mail/b/%s/imap/' % email
        imap = imaplib.IMAP4_SSL('imap.googlemail.com')
        imap.authenticate(url, consumer, token)
        
        # Get message count
        resp, count = imap.select(mailbox_to_scrape)
        # Get only seen messages
        resp, messages = imap.search(None, 'SEEN')
        messages = messages[0].split()
        # TODO: scrape_gmail_messages.update_state(state=(0, len(messages)))
        print "Message count: %d" % len(messages)
        
        # Find the phone numbers in each message
        for index in range(len(messages)):
            if index % 100 == 0:
                print 'Message %s/%s (%s numbers so far)' % (index, len(messages), len(phone_numbers))
            try:
                phone_numbers += find_phone_numbers(imap, messages[index])
                # TODO: scrape_gmail_messages.update_state(state=(index + 1, len(messages)))
            except:
                print 'Error: could not parse message #%s. Skipping.' % index
                from traceback import format_exc
                print format_exc()
                print
        
        imap.logout()
        end_datetime = datetime.now()
        
        # Log the process and its performance
        if config:
            try:
                conn = psycopg2.connect(
                    host = config['postgres']['host'],
                    port = int(config['postgres']['port']),
                    user = config['postgres']['username'],
                    password = config['postgres']['password'],
                    database = config['postgres']['database'],
                )
                cur = conn.cursor()
                try:
                    cur.execute("INSERT INTO processed (message_count, phone_number_count, start_time, end_time) VALUES (%s, %s, %s, %s)", (len(messages), len(phone_numbers), start_datetime, end_datetime))
                except psycopg2.ProgrammingError:
                    # Error, reset the connection
                    conn.rollback()
                    # Add table and retry
                    cur.execute("CREATE TABLE processed (message_count integer, phone_number_count integer, start_time timestamp, end_time timestamp);")
                    cur.execute("INSERT INTO processed (message_count, phone_number_count, start_time, end_time) VALUES (%s, %s, %s, %s)", (len(messages), len(phone_numbers), start_datetime, end_datetime))
                    print 'Database table "processed" created.'
                conn.commit()
                cur.close()
            except:
                print 'Error: could not log performance.'
                from traceback import format_exc
                print format_exc()
            finally:
                conn.close()
        else:
            print 'Processed %s: Phone Numbers = %s, Start = %s, End = %s' % (len(messages), len(phone_numbers), start_datetime.strftime('%m/%d/%Y %I:%M:%S %p'), end_datetime.strftime('%m/%d/%Y %I:%M:%S %p'))
        
        # Send completion email
        if app_email_info:
            send_email(app_email_info, email, 'Your DijScrape has finished', 'Hi, we are sending this message to inform you that your DijScrape has finished! To see your phone numbers, head back over to http://www.dijscrape.com/')
        
        return phone_numbers
    except:
        from traceback import format_exc
        exc = format_exc()
        print exc
        if not debug and error_email_info:
            send_email(error_email_info, admins, 'DijScrape Task Failed', exc)
        return phone_numbers
Example #34
0
 #send back charge request now
 request_xml = GC_CHARGE_SHIP_ORDER % (order_number,currency,total)
 logging.debug('gccharge send to goodle checkout: %s'%request_xml)
 try:
     result = urlfetch.fetch(url=GC_REQUEST_URLS[GC_WHICH],payload=request_xml,method=urlfetch.POST,headers=GC_HEADER);
 except Exception,e:
     logging.exception(e)
     web.error(500)  #will cause resend to google checkout
 try:
     if result.content.find('<charge-amount-notification') > 0:
         #charged, so save in ds
         data = get_google_notification_data(result.content)
         if 'original_buyer' not in data:
             ers = 'merchange-private-data not available in charge-amount-notification'
             logging.error(ers)
             helper.send_email('Suinova test payment failure notification',ers)
             web.succeed()    #return 200 not to resend to Google Checkout
             return
         buyer_id = data['original_buyer']
         buyer = helper.get_user_by_key(buyer_id,False)    #'gg_%s' % data['buyer'])
         if buyer is None:
             logging.warning('Buyer %s is not user!'%buyer_id)
             helper.send_email('Suinova test payment failure notification','Buyer %s not found'%buyer_id)
             web.succeed()   #return 200 not to resend to Google Checkout
             return
         gcfee = extract_text_in_tag(result.content,'total')
         if len(gcfee)>0:
             try:
                 fee = float(gcfee[0])
             except TypeError,e:
                 logging.error('float(%s) error:%s'%(gcfee[0],e))
Example #35
0
def main(
    start_time=dt.datetime(2005, 12, 1),
    end_time=dt.datetime(2020, 1, 1),
    in_dir_fmt='/project/superdarn/data/rawacf/%Y/%m/',
    fit_dir_fmt='/project/superdarn/data/fitacf/%Y/%m/',
    out_dir_fmt='/project/superdarn/data/netcdf/%Y/%m/',
    step=1,  # month
    skip_existing=True,
    fit_ext='*.fit',
):
    
    # Send the output to a log file
    original_stdout = sys.stdout
    if SAVE_OUTPUT_TO_LOGFILE:
        f = open('/homes/superdarn/logs/rawACF_to_netCDF_logs/raw_to_fit_to_net_{startDate}-{endDate}.log'.format(startDate = start_time.strftime("%Y%m%d"), endDate = end_time.strftime("%Y%m%d")), 'w')
        sys.stdout = f
    
    rstpath = os.getenv('RSTPATH')
    assert rstpath, 'RSTPATH environment variable needs to be set'
    hdw_dat_dir = os.path.join(rstpath, 'tables/superdarn/hdw/')
    
    # Running raw to NC
    radar_info = get_radar_params(hdw_dat_dir)
    run_dir = '/project/superdarn/run/%s' % get_random_string(4)
    if in_dir_fmt:
        raw_to_fit(start_time, end_time, run_dir, in_dir_fmt, fit_dir_fmt, MAKE_FIT_VERSIONS)

    # Loop over fit files in the monthly directories
    time = start_time
    while time <= end_time:
        # Set up directories
        out_dir = time.strftime(out_dir_fmt)
        print('Trying to make %s' % out_dir)
        os.makedirs(out_dir, exist_ok=True)

        # Loop over the files
        fit_fn_fmt = time.strftime(fit_dir_fmt)
        temp = f'fit_fn_fmt: {fit_fn_fmt}'
        print(temp)
        print(os.path.join(fit_fn_fmt, fit_ext))
        fit_fnames = glob.glob(os.path.join(fit_fn_fmt, fit_ext))
        print('Processing %i %s files in %s on %s' % (len(fit_fnames), fit_ext, fit_fn_fmt, time.strftime('%Y/%m')))
        for fit_fn in fit_fnames:
        
            # Check the file is big enough to be worth bothering with
            fn_info = os.stat(fit_fn)
            if fn_info.st_size < MIN_FITACF_FILE_SIZE:
                print('\n\n%s %1.1f MB\nFile too small - skipping' % (fit_fn, fn_info.st_size / 1E6))
                continue
            print('\n\nStarting from %s' % fit_fn)

            fn_head = '.'.join(os.path.basename(fit_fn).split('.')[:-1])
            out_fn = os.path.join(out_dir, '%s.nc' % fn_head)
            if os.path.isfile(out_fn):
                if skip_existing: 
                    print('%s exists - skipping' % out_fn)
                    continue
                else:
                    print('%s exists - deleting' % out_fn)
                    os.remove(out_fn)

            # Convert the fitACF to a netCDF
            radar_code = os.path.basename(fit_fn).split('.')[1]
            radar_info_t = id_hdw_params_t(time, radar_info[radar_code])

            status = fit_to_nc(time, fit_fn, out_fn, radar_info_t)

            if status == MULTIPLE_BEAM_DEFS_ERROR_CODE:
                print('Failed to convert {fitacfFile} because it had multiple beam definitions'.format(fitacfFile = fit_fn))
                continue
            elif status > 0:
                print('Failed to convert {fitacfFile}'.format(fitacfFile = fit_fn))
                continue

            print('Wrote output to %s' % out_fn)
        
        month = time.strftime('%Y%m')
        multiBeamLogDir = time.strftime(helper.FIT_NET_LOG_DIR) + month
        multiBeamFile = '{dir}/multi_beam_defs_{m}.log'.format(dir = multiBeamLogDir, m = month)
        if os.path.exists(multiBeamFile):
            subject = '"Multiple Beam Definitions Found - {date}"'.format(date = time.strftime('%Y/%m'))
            body = 'Files with multiple beam definitions have been found. See details in {file}'.format(file = multiBeamFile)
            helper.send_email(subject, body)

        time = add_months(time, step)  # time += dt.timedelta(months=1) doesn't exist
    sys.stdout = original_stdout
Example #36
0
def ppapprove(web,args=None):
    """ Called by PayPal through RETURNURL set in SetExpressCheckout command.
        Currently payment is done directly, a better way is to return a review page for the user to confirm payment and
        then come back to send DoExpressCheckoutPayment. But the current method is dirty quick.
        The L_NAME0 items are not displayed in the sandbox PayPal continue page, not sure whether it's the same on production site.
    """
    token = web.request.get('token')
    buyerid = web.request.get('PayerID')
    pkg = web.request.get('pkg')
    ukey = web.request.get('uid')
    if pkg == '':
        cs = web.request.get('CUSTOM')
        if cs.find(':') > 0:
            pkg,ukey = cs.split(':')
    logging.debug('pay.ppapprove: token=%s,buyerid=%s,pkg=%s,ukey=%s'%(token,buyerid,pkg,ukey))
    pg = gPackages[pkg.lower()]
    request = {'USER':PP_USER,'PWD':PP_PWD,'SIGNATURE':PP_SIG,'VERSION':PP_VER}
    request['PAYERID'] = buyerid
    request['TOKEN'] = token
    request['PAYMENTACTION'] = 'Sale'
    request['AMT'] = '%0.2f'%float(pg['price'])
    request['CURRENCYCODE'] = 'GBP'
    request['METHOD'] = 'DoExpressCheckoutPayment'
    request['CUSTOM'] = '%s:%s' % (ukey,pkg) # does this return?
    request['DESC'] = '%s. %s' % (pg['item'],pg['description'])
    request['L_NAME0'] = pg['item']
    request['L_NUMBER0'] = pg['id']
    request['L_DESC0'] = pg['description']
    request['L_AMT0'] = request['AMT']
    request['L_QTY0'] = '1'
    requests = urllib.urlencode(request)
    logging.debug('About to send to PayPal: %s'%requests)
    try:
        result = urlfetch.fetch(url=PP_ECURL,payload=requests,method=urlfetch.POST,headers={'Content-type':'application/x-www-form-urlencoded'})
        if result.content.find('ACK=Success')>=0:
            data = cgi.parse_qs(result.content)
            #token = data['TOKEN'][-1]
            payment_status = data['PAYMENTSTATUS'][0]
            if payment_status == 'Completed':
                data2 = {'method':'PP','quantity':'1','item_id':pkg,'buyer':buyerid}
                data2['order_number'] = data['TRANSACTIONID'][0]
                data2['price'] = data['AMT'][0]
                data2['currency'] = data['CURRENCYCODE'][0]
                if 'SETTLEAMT' in data:
                    logging.debug('settlement: %s'%data['SETTLEAMT'][0])
                fee = float(data['FEEAMT'][0])
                tstamp = datetime_from_timestampz(data['ORDERTIME'][0])
                buyer = helper.get_user_by_key(ukey,False)
                if buyer is None:
                    logging.warning('Buyer %s is not user!'%ukey)
                    helper.send_email('Suinova test payment failure notification','Buyer %s not found'%ukey)
                    web.response.out.write('Thank you!')
                    return
                try:
                    save_exchange(buyer,tstamp,fee,data2)
                except Exception:
                    pass
                web.response.out.write(THANKU_PAGE)
            else:
                logging.warning('ppapprove result from DoExpressCheckoutPayment, payment status is %s'%payment_status)
                logging.info(result.content)
                web.response.out.write('Not completed')
        else:
            logging.error('ppapprove send DoExpressCheckoutPayment returned: %s'%result.content)
            web.fail('Error contacting PayPal, try later, <a href="/">Go back</a>')
    except Exception,e:
        logging.exception(e)
        web.fail('Error contacting PayPal, try later, <a href="/">Go back</a>')