Example #1
0
def _getConnection():
    i = 0
    count = 5
    while 1:
        try:
            i = i + 1
            conn = pymysql.connect(**settings.db)
            return conn
        except pymysql.Error as e:
            print("Error %d: %s" % (e.args[0], e.args[1]))
            if i >= 10:
                print("sql connection get count %d " % (count))
                return None
            time.sleep(5)
Example #2
0
def lambda_handler(event, context):
    #print("Received event: " + json.dumps(event, indent=2))

    try:
        conn = pymysql.connect(rds_host,
                               user=db_username,
                               passwd=db_password,
                               db=db_name,
                               connect_timeout=5)
    except pymysql.MySQLError as e:
        print(
            "ERROR: Unexpected error: Could not connect to MySQL instance '{}', db '{}' with user '{}'"
            .format(rds_host, db_name, db_username))
        raise e

    #print('Connected to MySQL instance \'{}\', db \'{}\' with user \'{}\''.format(rds_host, db_name, db_username))

    # Get the object from the event and show its content type
    bucket = event['Records'][0]['s3']['bucket']['name']
    key = urllib.parse.unquote_plus(event['Records'][0]['s3']['object']['key'],
                                    encoding='utf-8')
    try:
        obj = s3.get_object(Bucket=bucket, Key=key)
        regex = re.compile(r'\.\w+$')
        exts = regex.findall(key)

        if len(exts) > 0:
            if (exts[0] == '.json'):
                j = json.loads(obj['Body'].read())
                process_json(conn, j)
            elif (exts[0] == '.csv'):
                rows = csv.reader(obj['Body'].read().split('\n'),
                                  delimiter=',')
                process_csv(conn, rows)
            else:
                print('Unrecognised file format {}'.format(exts[0]))
                return False
        else:
            print('No extension on file {}'.format(key))

        return True
    except Exception as e:
        print(e)
        print(
            'Error getting object {} from bucket {}. Make sure they exist and your bucket is in the same region as this function.'
            .format(key, bucket))
        raise e
Example #3
0
    def VTiter(self, *parsedArgs, **envars):
        from lib import pymysql
        t = pymysql.FIELD_TYPE

        typetrans = {
            t.DECIMAL: 'INT',
            t.TINY: 'INT',
            t.SHORT: 'INT',
            t.LONG: 'INT',
            t.FLOAT: 'REAL',
            t.DOUBLE: 'REAL',
            t.NULL: '',
            t.TIMESTAMP: 'TEXT',
            t.LONGLONG: 'TEXT',
            t.INT24: 'TEXT',
            t.DATE: 'TEXT',
            t.TIME: 'TEXT',
            t.DATETIME: 'TEXT',
            t.YEAR: 'INT',
            t.NEWDATE: 'TEXT',
            t.VARCHAR: 'TEXT',
            t.BIT: 'INT',
            t.NEWDECIMAL: 'INT',
            t.ENUM: 'TEXT',
            t.SET: 'TEXT',
            t.TINY_BLOB: 'TEXT',
            t.MEDIUM_BLOB: 'TEXT',
            t.LONG_BLOB: 'TEXT',
            t.BLOB: 'TEXT',
            t.VAR_STRING: 'TEXT',
            t.STRING: 'TEXT',
            t.GEOMETRY: 'TEXT'
        }

        largs, dictargs = self.full_parse(parsedArgs)

        if 'query' not in dictargs:
            raise functions.OperatorError(__name__.rsplit('.')[-1], "No query argument ")

        query = dictargs['query']

        host = dictargs.get('host', dictargs.get('h', '127.0.0.1'))
        port = int(dictargs.get('port', 3306))
        user = dictargs.get('user', dictargs.get('u', 'root'))
        passwd = dictargs.get('passwd', dictargs.get('p', ''))
        db = dictargs.get('db', 'mysql')

        try:
            conn = pymysql.connect(host=host, port=port, user=user, passwd=passwd, db=db, use_unicode=True)

            cur = conn.cursor(pymysql.cursors.SSCursor)
            cur.execute(query)

            desc = cur.description
            if desc == None:
                yield [('None',)]
            else:
                yield [(c[0], typetrans.get(c[1], '')) for c in desc]

            for i in cur:
                yield [unicode(c) if type(c) not in (long, int, float, str, unicode, NoneType, bool) else c for c in i]

        except (pymysql.err.InternalError, pymysql.err.ProgrammingError) as e:
            raise functions.OperatorError(__name__.rsplit('.')[-1], str(e[0]) + ': ' + e[1])
        except Exception, e:
            raise functions.OperatorError(__name__.rsplit('.')[-1], str(e))
Example #4
0
    def VTiter(self, *parsedArgs, **envars):
        from lib import pymysql
        t = pymysql.FIELD_TYPE

        typetrans = {
            t.DECIMAL: 'INT',
            t.TINY: 'INT',
            t.SHORT: 'INT',
            t.LONG: 'INT',
            t.FLOAT: 'REAL',
            t.DOUBLE: 'REAL',
            t.NULL: '',
            t.TIMESTAMP: 'TEXT',
            t.LONGLONG: 'TEXT',
            t.INT24: 'TEXT',
            t.DATE: 'TEXT',
            t.TIME: 'TEXT',
            t.DATETIME: 'TEXT',
            t.YEAR: 'INT',
            t.NEWDATE: 'TEXT',
            t.VARCHAR: 'TEXT',
            t.BIT: 'INT',
            t.NEWDECIMAL: 'INT',
            t.ENUM: 'TEXT',
            t.SET: 'TEXT',
            t.TINY_BLOB: 'TEXT',
            t.MEDIUM_BLOB: 'TEXT',
            t.LONG_BLOB: 'TEXT',
            t.BLOB: 'TEXT',
            t.VAR_STRING: 'TEXT',
            t.STRING: 'TEXT',
            t.GEOMETRY: 'TEXT'
        }

        largs, dictargs = self.full_parse(parsedArgs)

        if 'query' not in dictargs:
            raise functions.OperatorError(
                __name__.rsplit('.')[-1], "No query argument ")

        query = dictargs['query']

        host = dictargs.get('host', dictargs.get('h', '127.0.0.1'))
        port = int(dictargs.get('port', 3306))
        user = dictargs.get('user', dictargs.get('u', 'root'))
        passwd = dictargs.get('passwd', dictargs.get('p', ''))
        db = dictargs.get('db', 'mysql')

        try:
            conn = pymysql.connect(host=host,
                                   port=port,
                                   user=user,
                                   passwd=passwd,
                                   db=db,
                                   use_unicode=True)

            cur = conn.cursor(pymysql.cursors.SSCursor)
            cur.execute(query)

            desc = cur.description
            if desc == None:
                yield [('None', )]
            else:
                yield [(c[0], typetrans.get(c[1], '')) for c in desc]

            for i in cur:
                yield [
                    unicode(c) if type(c) not in (long, int, float, str,
                                                  unicode, NoneType,
                                                  bool) else c for c in i
                ]

        except (pymysql.err.InternalError, pymysql.err.ProgrammingError) as e:
            raise functions.OperatorError(
                __name__.rsplit('.')[-1],
                str(e[0]) + ': ' + e[1])
        except Exception, e:
            raise functions.OperatorError(__name__.rsplit('.')[-1], str(e))