def _getConnection(): i = 0 count = 5 while 1: try: i = i + 1 conn = pymysql.connect(**settings.db) return conn except pymysql.Error as e: print("Error %d: %s" % (e.args[0], e.args[1])) if i >= 10: print("sql connection get count %d " % (count)) return None time.sleep(5)
def lambda_handler(event, context): #print("Received event: " + json.dumps(event, indent=2)) try: conn = pymysql.connect(rds_host, user=db_username, passwd=db_password, db=db_name, connect_timeout=5) except pymysql.MySQLError as e: print( "ERROR: Unexpected error: Could not connect to MySQL instance '{}', db '{}' with user '{}'" .format(rds_host, db_name, db_username)) raise e #print('Connected to MySQL instance \'{}\', db \'{}\' with user \'{}\''.format(rds_host, db_name, db_username)) # Get the object from the event and show its content type bucket = event['Records'][0]['s3']['bucket']['name'] key = urllib.parse.unquote_plus(event['Records'][0]['s3']['object']['key'], encoding='utf-8') try: obj = s3.get_object(Bucket=bucket, Key=key) regex = re.compile(r'\.\w+$') exts = regex.findall(key) if len(exts) > 0: if (exts[0] == '.json'): j = json.loads(obj['Body'].read()) process_json(conn, j) elif (exts[0] == '.csv'): rows = csv.reader(obj['Body'].read().split('\n'), delimiter=',') process_csv(conn, rows) else: print('Unrecognised file format {}'.format(exts[0])) return False else: print('No extension on file {}'.format(key)) return True except Exception as e: print(e) print( 'Error getting object {} from bucket {}. Make sure they exist and your bucket is in the same region as this function.' .format(key, bucket)) raise e
def VTiter(self, *parsedArgs, **envars): from lib import pymysql t = pymysql.FIELD_TYPE typetrans = { t.DECIMAL: 'INT', t.TINY: 'INT', t.SHORT: 'INT', t.LONG: 'INT', t.FLOAT: 'REAL', t.DOUBLE: 'REAL', t.NULL: '', t.TIMESTAMP: 'TEXT', t.LONGLONG: 'TEXT', t.INT24: 'TEXT', t.DATE: 'TEXT', t.TIME: 'TEXT', t.DATETIME: 'TEXT', t.YEAR: 'INT', t.NEWDATE: 'TEXT', t.VARCHAR: 'TEXT', t.BIT: 'INT', t.NEWDECIMAL: 'INT', t.ENUM: 'TEXT', t.SET: 'TEXT', t.TINY_BLOB: 'TEXT', t.MEDIUM_BLOB: 'TEXT', t.LONG_BLOB: 'TEXT', t.BLOB: 'TEXT', t.VAR_STRING: 'TEXT', t.STRING: 'TEXT', t.GEOMETRY: 'TEXT' } largs, dictargs = self.full_parse(parsedArgs) if 'query' not in dictargs: raise functions.OperatorError(__name__.rsplit('.')[-1], "No query argument ") query = dictargs['query'] host = dictargs.get('host', dictargs.get('h', '127.0.0.1')) port = int(dictargs.get('port', 3306)) user = dictargs.get('user', dictargs.get('u', 'root')) passwd = dictargs.get('passwd', dictargs.get('p', '')) db = dictargs.get('db', 'mysql') try: conn = pymysql.connect(host=host, port=port, user=user, passwd=passwd, db=db, use_unicode=True) cur = conn.cursor(pymysql.cursors.SSCursor) cur.execute(query) desc = cur.description if desc == None: yield [('None',)] else: yield [(c[0], typetrans.get(c[1], '')) for c in desc] for i in cur: yield [unicode(c) if type(c) not in (long, int, float, str, unicode, NoneType, bool) else c for c in i] except (pymysql.err.InternalError, pymysql.err.ProgrammingError) as e: raise functions.OperatorError(__name__.rsplit('.')[-1], str(e[0]) + ': ' + e[1]) except Exception, e: raise functions.OperatorError(__name__.rsplit('.')[-1], str(e))
def VTiter(self, *parsedArgs, **envars): from lib import pymysql t = pymysql.FIELD_TYPE typetrans = { t.DECIMAL: 'INT', t.TINY: 'INT', t.SHORT: 'INT', t.LONG: 'INT', t.FLOAT: 'REAL', t.DOUBLE: 'REAL', t.NULL: '', t.TIMESTAMP: 'TEXT', t.LONGLONG: 'TEXT', t.INT24: 'TEXT', t.DATE: 'TEXT', t.TIME: 'TEXT', t.DATETIME: 'TEXT', t.YEAR: 'INT', t.NEWDATE: 'TEXT', t.VARCHAR: 'TEXT', t.BIT: 'INT', t.NEWDECIMAL: 'INT', t.ENUM: 'TEXT', t.SET: 'TEXT', t.TINY_BLOB: 'TEXT', t.MEDIUM_BLOB: 'TEXT', t.LONG_BLOB: 'TEXT', t.BLOB: 'TEXT', t.VAR_STRING: 'TEXT', t.STRING: 'TEXT', t.GEOMETRY: 'TEXT' } largs, dictargs = self.full_parse(parsedArgs) if 'query' not in dictargs: raise functions.OperatorError( __name__.rsplit('.')[-1], "No query argument ") query = dictargs['query'] host = dictargs.get('host', dictargs.get('h', '127.0.0.1')) port = int(dictargs.get('port', 3306)) user = dictargs.get('user', dictargs.get('u', 'root')) passwd = dictargs.get('passwd', dictargs.get('p', '')) db = dictargs.get('db', 'mysql') try: conn = pymysql.connect(host=host, port=port, user=user, passwd=passwd, db=db, use_unicode=True) cur = conn.cursor(pymysql.cursors.SSCursor) cur.execute(query) desc = cur.description if desc == None: yield [('None', )] else: yield [(c[0], typetrans.get(c[1], '')) for c in desc] for i in cur: yield [ unicode(c) if type(c) not in (long, int, float, str, unicode, NoneType, bool) else c for c in i ] except (pymysql.err.InternalError, pymysql.err.ProgrammingError) as e: raise functions.OperatorError( __name__.rsplit('.')[-1], str(e[0]) + ': ' + e[1]) except Exception, e: raise functions.OperatorError(__name__.rsplit('.')[-1], str(e))