def main_handler(event, context): logger.info("start main handler") if "Records" not in event.keys(): return {"errorMsg": "event is not come from cos"} #初始化COS及获取COS文件信息 appid = event['Records'][0]['cos']['cosBucket']['appid'] bucket = event['Records'][0]['cos']['cosBucket']['name'] + '-' + str(appid) key = event['Records'][0]['cos']['cosObject']['key'] key = key.replace('/' + str(appid) + '/' + event['Records'][0]['cos']['cosBucket']['name'] + '/', '', 1) if key[-1] == '/': return {"errorMsg": "this is floder"} # 提取文件名 shotname folder_path,file_name = os.path.split(key) download_path = '/tmp/{}'.format(file_name) logger.info("File_name is " + key) logger.info("download_path is " + download_path) logger.info("Get from [%s] to download file [%s]" % (bucket, key)) #使用临时秘钥 secret_id = os.environ.get('TENCENTCLOUD_SECRETID') secret_key = os.environ.get('TENCENTCLOUD_SECRETKEY') token = os.environ.get('TENCENTCLOUD_SESSIONTOKEN') # 从COS下载需要同步的文件 cos_client_source = cosClient = CosS3Client(CosConfig(Region=source_region, SecretId=secret_id, SecretKey=secret_key,Token=token)) try: response = cos_client_source.get_object(Bucket=bucket, Key=key, ) response['Body'].get_stream_to_file(download_path) logger.info("Download file [%s] Success" % key) except CosServiceError as e: logger.info("Download file [%s] Fail" % key) logger.error(e.get_error_code()) logger.error(e.get_error_msg()) logger.error(e.get_resource_location()) delete_local_file(str(download_path)) return ("Download file [%s] from [%s] fail" % (key,bucket)) # 同步文件到异地COS bucket logger.info("Start to upload file to target_bucket") cos_client_target = cosClient = CosS3Client(CosConfig(Region=target_region, SecretId=target_secret_id, SecretKey=target_secret_key,)) try: response = cos_client_target.put_object_from_local_file( Bucket=target_bucket, LocalFilePath=download_path, Key=key) logger.info("Upload result is [%s]" % response) except CosServiceError as e: logger.error(e.get_error_code()) logger.error(e.get_error_msg()) logger.error(e.get_resource_location()) delete_local_file(str(download_path)) return ("Upload file [%s] from [%s] fail" % (key,bucket)) delete_local_file(str(download_path)) return "Sync File Success"
def main_handler(event, context): logger.info("start unpack template function") secret_id = os.getenv('TENCENTCLOUD_SECRETID') secret_key = os.getenv('TENCENTCLOUD_SECRETKEY') token = os.getenv('TENCENTCLOUD_SESSIONTOKEN') config = CosConfig(Secret_id=secret_id, Secret_key=secret_key, Region=region, Token=token) client = CosS3Client(config) for record in event['Records']: try: appid = record['cos']['cosBucket']['appid'] bucket = record['cos']['cosBucket']['name'] + '-' + appid filename = os.path.basename(record['cos']['cosObject']['url']) download_path = '/tmp/{}'.format(filename.encode('gb18030')) key = record['cos']['cosObject']['key'] key = key.replace('/' + appid + '/' + record['cos']['cosBucket']['name'] + '/', '', 1) # 创建本地解压路径 isExists = os.path.exists('/tmp/unpack') if not isExists: os.mkdir('/tmp/unpack') unpack_path = '/tmp/unpack' # 提取文件名 shotname (filepath, tempfilename) = os.path.split(filename); (shotname, extension) = os.path.splitext(tempfilename); if extension[1:] not in unpack_suffix.split(','): logger.info("object suffix is [%s], expected: [%s]", extension, unpack_suffix) return "object suffix is [%s], expected: [%s]" % (extension, unpack_suffix) logger.info("object name is [%s]", shotname) # download rar from cos logger.info("get from [%s] to download object [%s]", bucket, filename) try: response = client.get_object(Bucket=bucket, Key=key, ) response['Body'].get_stream_to_file(download_path) logger.info("download object [%s] Success", filename) except CosServiceError as e: print(e.get_error_code()) print(e.get_error_msg()) print(e.get_resource_location()) logger.info("download object [%s] failed", filename) return "download object fail" # start to extract archive file and upload to bucket_upload logger.info("start to extract archive file") Archive(download_path).extractall(unpack_path, auto_create_dir=True) logger.info("extract success") upload_local_file(client, '/tmp/unpack', target_dir) # clean files delete_local_file(str(download_path)) delete_local_file(str(unpack_path)) return "extract and upload success" except Exception as e: print(e) raise e return "extract and upload fail"
def main_handler(event, context): logger.info("start main handler") if "Records" not in event.keys(): return {"errorMsg": "event is not come from cos"} secret_id = os.environ.get('TENCENTCLOUD_SECRETID') # 使用环境变量中的 secretId secret_key = os.environ.get('TENCENTCLOUD_SECRETKEY') # 使用环境变量中的 secretKey token = os.environ.get('TENCENTCLOUD_SESSIONTOKEN') config = CosConfig( Region=region, SecretId=secret_id, SecretKey=secret_key, Token=token, ) cos_client = CosS3Client(config) start_time = datetime.datetime.now() res = map_caller(event, context, cos_client) end_time = datetime.datetime.now() print("data mapping duration: " + str((end_time - start_time).microseconds / 1000) + "ms") if res == 0: return "Data mapping SUCCESS" else: return "Data mapping FAILED"
def __init__(self, kafka_instance_id, topic_name, topic_id, kafka_address, bucket_address, partition_max_to_cos_bytes, partition_max_timeout_ms, partition_id, consumer_timeout_ms, group_id, offset_type, region, secret_id, secret_key, token=None): self.topic_name = topic_name self.topic_id = topic_id self.kafka_address = kafka_address self.bucket_address = bucket_address self.kafka_instance_id = kafka_instance_id self.partition_max_to_cos_bytes = partition_max_to_cos_bytes self.partition_max_timeout_ms = partition_max_timeout_ms self.partition_id = partition_id self.consumer_timeout_ms = consumer_timeout_ms self.group_id = group_id self.offset_type = offset_type config = CosConfig(Region=region, SecretId=secret_id, SecretKey=secret_key, Token=token) # 获取配置对象 self.client = CosS3Client(config)
def getPresignedUrl(event, context): print('event', event) print('context', context) body = json.loads(event['body']) # 可以通过客户端传来的token进行鉴权,只有鉴权通过才可以获得临时上传地址 # 这一部分可以按需修改,例如用户的token可以在redis获取,可以通过某些加密方法获取等 # 也可以是传来一个username和一个token,然后去数据库中找这个username对应的token是否 # 与之匹配等,这样会尽可能的提升安全性 if "key" not in body or "token" not in body or body['token'] != 'mytoken' or "key" not in body: return {"url": None} # 初始化COS对象 region = os.environ.get("region") secret_id = os.environ.get("TENCENTCLOUD_SECRETID") secret_key = os.environ.get("TENCENTCLOUD_SECRETKEY") token = os.environ.get("TENCENTCLOUD_SESSIONTOKEN") config = CosConfig(Region=region, SecretId=secret_id, SecretKey=secret_key, Token=token) client = CosS3Client(config) response = client.get_presigned_url( Method='PUT', Bucket=os.environ.get('bucket_name'), Key=body['key'], Expired=30, ) return {"url": response.split("?sign=")[0], "sign": urllib.parse.unquote(response.split("?sign=")[1]), "token": os.environ.get("TENCENTCLOUD_SESSIONTOKEN")}
def uploadToScf(event, context): print('event', event) print('context', context) body = json.loads(event['body']) # 可以通过客户端传来的token进行鉴权,只有鉴权通过才可以获得临时上传地址 # 这一部分可以按需修改,例如用户的token可以在redis获取,可以通过某些加密方法获取等 # 也可以是传来一个username和一个token,然后去数据库中找这个username对应的token是否 # 与之匹配等,这样会尽可能的提升安全性 if "key" not in body or "token" not in body or body['token'] != 'mytoken' or "key" not in body: return {"url": None} pictureBase64 = body["picture"].split("base64,")[1] with open('/tmp/%s' % body['key'], 'wb') as f: f.write(base64.b64decode(pictureBase64)) region = os.environ.get("region") secret_id = os.environ.get("TENCENTCLOUD_SECRETID") secret_key = os.environ.get("TENCENTCLOUD_SECRETKEY") token = os.environ.get("TENCENTCLOUD_SESSIONTOKEN") config = CosConfig(Region=region, SecretId=secret_id, SecretKey=secret_key, Token=token) client = CosS3Client(config) response = client.upload_file( Bucket=os.environ.get("bucket_name"), LocalFilePath='/tmp/%s' % body['key'], Key=body['key'], ) return { "uploaded": 1, "url": 'https://%s.cos.%s.myqcloud.com' % ( os.environ.get("bucket_name"), os.environ.get("region")) + body['key'] }
def main_handler(event, context): secret_id = os.environ.get('TENCENTCLOUD_SECRETID') # Using the secterId in environment variables. 使用环境变量中的 secretId secret_key = os.environ.get('TENCENTCLOUD_SECRETKEY') # Using the secretKey in environment variables. 使用环境变量中的 secretKey token = os.environ.get('TENCENTCLOUD_SESSIONTOKEN') config = CosConfig(Region=REGION, SecretId=secret_id, SecretKey=secret_key, Token=token,) cos_client = CosS3Client(config) connection = pymysql.connect(host=DB_HOST, user=DB_USER, password=DB_USER_PASSWORD, port=int(DB_PORT), db=DB_NAME, charset='utf8', cursorclass=pymysql.cursors.DictCursor) with connection.cursor() as cursor: sql = 'show databases' cursor.execute(sql) res = cursor.fetchall() print res timestr = time.strftime('%Y%m%d-%H%M%S', time.localtime(time.time())) filename = DB_NAME+"-"+timestr+".sql" filepath = BACKUP_PATH+os.sep+filename print ("Start Backup") dumpcmd = "./mysqldump -h" + DB_HOST + " -P" + DB_PORT + " -u" + DB_USER + " -p" + DB_USER_PASSWORD + " " + DB_NAME + " > " + filepath print (dumpcmd) print os.popen(dumpcmd).read() print ("Backup script completed") print ("Your backups has been created in '" + filepath + "' file") print (os.popen('ls -l /tmp').read()) print ("finish backup") print ("start send to cos") backup2cos(cos_client,filepath,BACKUP_BUCKET,"/"+filename) print ("finish send to cos")
def __init__(self, *args, **kwargs): appid = int(kwargs['appid']) region = kwargs['region'] accesskeyid = unicode(kwargs['accesskeyid']) accesskeysecret = unicode(kwargs['accesskeysecret']) bucket = unicode(kwargs['bucket']) if 'prefix_dir' in kwargs: self._prefix_dir = kwargs['prefix_dir'] else: self._prefix_dir = None if 'part_size' in kwargs: self._part_size = kwargs['part_size'] else: self._part_size = 1 conf = CosConfig(Appid=appid, Access_id=str(accesskeyid), Access_key=str(accesskeysecret), Region=region) self._cos_client = CosS3Client(conf=conf, retry=1) self._bucket = bucket self._max_retry = 20 self._appid = appid self._region = region self._accesskeyid = str(accesskeyid) self._accesskeysecret = str(accesskeysecret)
def write_to_cos(path, cos_path): """输出用户信息""" matchObj = re.match(r'https://(.*).cos.(.*).myqcloud.com(.*)', cos_path, re.M | re.I) dst_bucket = matchObj.group(1) dst_path = matchObj.group(3) region = matchObj.group(2) secret_id = os.environ.get('TENCENTCLOUD_SECRETID') secret_key = os.environ.get('TENCENTCLOUD_SECRETKEY') token = os.environ.get('TENCENTCLOUD_SESSIONTOKEN') cos_client = CosS3Client( CosConfig(Region=region, SecretId=secret_id, SecretKey=secret_key, Token=token)) cos_client.put_object_from_local_file(Bucket=dst_bucket, LocalFilePath=path, Key=dst_path + '/' + path.split('/')[-1]) delete_local_file(path) return {"file": dst_bucket + dst_path + '/' + path.split('/')[-1]}
def main_handler(event, context): logger.info("start main handler") if "Records" not in event.keys(): return {"errorMsg": "event is not come from cos"} secret_id = os.environ.get( 'TENCENT_SECRET_ID' ) # Using the secterId in environment variables. 使用环境变量中的 secretId secret_key = os.environ.get( 'TENCENT_SECRET_KEY' ) # Using the secretKey in environment variables. 使用环境变量中的 secretKey config = CosConfig( Region=region, SecretId=secret_id, SecretKey=secret_key, ) cos_client = CosS3Client(config) start_time = datetime.datetime.now() res = reduce_caller(event, context, cos_client) end_time = datetime.datetime.now() print("data reducing duration: " + str((end_time - start_time).microseconds / 1000) + "ms") if res == 0: return "Data reducing SUCCESS" else: return "Data reducing FAILED"
def __init__(self, config): cos_config = CosConfig( Region=config['cos_region'], Secret_id=config['secret_id'], Secret_key=config['secret_key'], Token=config.get('token', None), ) self.cos_client = CosS3Client(cos_config) self.cos_bucket = config['cos_bucket'] self.cos_key = config['cos_key'] self.url = config['url']
def main_handler(event, context): logger.info("start main handler") #使用临时秘钥初始化COS Client secret_id = os.environ.get('TENCENTCLOUD_SECRETID') secret_key = os.environ.get('TENCENTCLOUD_SECRETKEY') token = os.environ.get('TENCENTCLOUD_SESSIONTOKEN') cosClient = CosS3Client( CosConfig(Region=region, SecretId=secret_id, SecretKey=secret_key, Token=token)) #判断请求是否从API网关传递,通过网关传递源文件下载地址,在body中获取 if "body" in event.keys(): download_path = event['body'] key = download_path.split('/')[-1] logger.info('api download_path: ' + download_path) #判断请求是否从COS传递 elif "Records" in event.keys(): #从event里获取COS文件信息,并获取下载地址 bucket = event['Records'][0]['cos']['cosBucket']['name'] + '-' + event[ 'Records'][0]['cos']['cosBucket']['appid'] key = "/".join( event['Records'][0]['cos']['cosObject']['key'].split("/")[3:]) download_path = event['Records'][0]['cos']['cosObject']['url'] logger.info('cos download_path: ' + download_path) else: return { "code": 410, "errorMsg": "event does not come from COS or APIGW" } logger.info('key: ' + key) upload_path = '/tmp/new-' + key.split('/')[-1] logger.info('upload_path: ' + upload_path) # 执行ffmpeg命令,从下载地址读流的方式进行转码 child = subprocess.run(video_press % (download_path, upload_path), stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True, shell=True) # 上传视频,可自定义上传路径 cosClient.put_object_from_local_file(Bucket=target_bucket, LocalFilePath=upload_path, Key=target_path + '/' + upload_path.split('/')[-1]) delete_local_file(upload_path) return 'success'
def __init__(self, start_time, request_id, kafka_instance_id, topic_name, topic_id, kafka_address, bucket_address, file_path_prefix, once_max_to_cos_bytes, function_timeout_ms, partition_id, group_id, offset_type, region, secret_id, secret_key, token=None): self.start_time = start_time self.request_id = request_id self.kafka_instance_id = kafka_instance_id self.topic_name = topic_name self.topic_id = topic_id self.kafka_address = kafka_address self.bucket_address = bucket_address self.file_path_prefix = file_path_prefix self.once_max_to_cos_bytes = once_max_to_cos_bytes self.function_timeout_ms = function_timeout_ms self.partition_id = partition_id self.group_id = group_id self.offset_type = offset_type config = CosConfig(Region=region, SecretId=secret_id, SecretKey=secret_key, Token=token) # 获取配置对象 self.cos_client = CosS3Client(config) self.cos_upload_speed = float(init_cos_upload_speed) self.start_offset = PyKafkaOffsetType.EARLIEST
def main_handler(event, context): logger.info("start main handler") if "Records" not in event.keys(): return {"errorMsg": "event is not come from cos"} logger.info("source_bucket is " + source_bucket) logger.info("target_bucket is " + target_bucket) #使用临时秘钥 secret_id = os.environ.get('TENCENTCLOUD_SECRETID') secret_key = os.environ.get('TENCENTCLOUD_SECRETKEY') token = os.environ.get('TENCENTCLOUD_SESSIONTOKEN') cosClient = CosS3Client( CosConfig(Region=region, SecretId=secret_id, SecretKey=secret_key, Token=token)) appid = event['Records'][0]['cos']['cosBucket']['appid'] key = event['Records'][0]['cos']['cosObject']['key'] key = key.replace( '/' + str(appid) + '/' + event['Records'][0]['cos']['cosBucket']['name'] + '/', '', 1) if key[-1] == '/': return {"errorMsg": "this is floder,do not need to sync"} logger.info("key:" + key) response = cosClient.copy_object(Bucket=target_bucket, Key=key, CopySource={ 'Bucket': source_bucket, 'Key': key, 'Region': region }) print("response is ", response) return 'copy success'
def __init__(self, config): self.config = config cred = credential.Credential(config['secret_id'], config['secret_key'], config['token']) httpProfile = HttpProfile() httpProfile.endpoint = "cdn.tencentcloudapi.com" clientProfile = ClientProfile() clientProfile.httpProfile = httpProfile cdnClient = cdn_client.CdnClient(cred, config['scf_region'], clientProfile) self.cdn_client = cdnClient self.cos_path = config['cos_path'] cos_config = CosConfig( Region=config['cos_region'], Secret_id=config['secret_id'], Secret_key=config['secret_key'], Token=config.get('token', None), ) self.cos_client = CosS3Client(cos_config) self.cos_bucket = config['cos_bucket']
appid = os.environ.get('appid') # 请在函数配置中添加环境变量appid,并填入您的 APPID secret_id = os.environ.get( 'secret_id') # 请在函数配置中添加环境变量secret_id,并填入您的 SecretId secret_key = os.environ.get( 'secret_key') # 请在函数配置中添加环境变量secret_key,并填入您的 SecretKey bucket_region = os.environ.get( 'bucket_region') # 请在函数配置中添加环境变量bucket_region,并填入您COS Bucket所在的地域 token = '' bucket_upload = os.environ.get( 'bucket_upload') # 请在函数配置中添加环境变量bucket_upload,并填入您要用来存放图片的Bucket名 config = CosConfig(Secret_id=secret_id, Secret_key=secret_key, Region=bucket_region, Token=token) client_cos = CosS3Client(config) def delete_local_file(src): logger.info("delete files and folders") if os.path.isfile(src): try: os.remove(src) except: pass elif os.path.isdir(src): for item in os.listdir(src): itemsrc = os.path.join(src, item) delete_file_folder(itemsrc) try: os.rmdir(src)
from qcloud_cos_v5 import CosConfig from qcloud_cos_v5 import CosS3Client from tencentcloud.common import credential from tbp import tbp_client, models as tbp_models from tts import tts_client, models as tts_models from tencentcloud.scf.v20180416 import scf_client, models as scf_models bot_id = os.environ.get('bot_id') bucket = os.environ.get('bucket') secret_id = os.environ.get('secret_id') secret_key = os.environ.get('secret_key') region = os.environ.get('region') wxtoken = os.environ.get('wxtoken') appid = os.environ.get('appid') secret = os.environ.get('secret') cosClient = CosS3Client( CosConfig(Region=region, SecretId=secret_id, SecretKey=secret_key)) scfClient = scf_client.ScfClient(credential.Credential(secret_id, secret_key), region) tbpClient = tbp_client.TbpClient(credential.Credential(secret_id, secret_key), region) ttsClient = tts_client.TtsClient(credential.Credential(secret_id, secret_key), region) key = 'news/content.json' indexKey = 'news/content_index.json' accessTokenKey = 'access/token.json' accessToken = None articlesList = None def getAccessToken():
def main_handler(event, context): logger.info("start main handler") if "requestContext" not in event.keys(): return {"code": 410, "errorMsg": "event is not come from api gateway"} if "body" not in event.keys(): return { "isBase64Encoded": False, "statusCode": 200, "headers": { "Content-Type": "text", "Access-Control-Allow-Origin": "*" }, "body": "there is no file from api gateway" } # save api gateway file to local temp file logger.info("Start to download images from APIGW") time = datetime.datetime.now() file_name = '{}'.format(time) + "-test.jpg" logger.info("file_name is : %s" % file_name) local_path = u'/tmp/{}'.format(file_name) logger.info("local_path is : %s" % local_path) with open(local_path, 'w') as wfile: wfile.write(base64.b64decode(event['body'])) # start to upload to cos secret_id = os.environ.get('TENCENTCLOUD_SECRETID') # 使用环境变量中的 secretId secret_key = os.environ.get('TENCENTCLOUD_SECRETKEY') # 使用环境变量中的 secretKey token = os.environ.get('TENCENTCLOUD_SESSIONTOKEN') config = CosConfig( Region=bucket_region, SecretId=secret_id, SecretKey=secret_key, Token=token, ) cos_client = CosS3Client(config) logger.info("Start to upload images to cos") res_cos = cos_client.put_object_from_local_file(Bucket=bucket_upload, LocalFilePath=local_path, Key='{}'.format(file_name)) logger.info("upload to cos result is : %s" % res_cos) # start to detection logger.info("Start to detection") try: cred = credential.Credential(secret_id, secret_key, token) httpProfile = HttpProfile() httpProfile.endpoint = "ocr.tencentcloudapi.com" clientProfile = ClientProfile() clientProfile.httpProfile = httpProfile client = ocr_client.OcrClient(cred, "ap-beijing", clientProfile) req = models.GeneralBasicOCRRequest() params = '{"ImageBase64":"%s"}' % event['body'] req.from_json_string(params) resp = client.GeneralBasicOCR(req) res_ai = json.loads(resp.to_json_string()) res_text = " " print len(res_ai["TextDetections"]) for i in range(len(res_ai["TextDetections"])): res_text = res_text + str( res_ai["TextDetections"][i]["DetectedText"].encode('utf-8')) except TencentCloudSDKException as err: print(err) delete_local_file(local_path) response = { "isBase64Encoded": False, "statusCode": 200, "headers": { "Content-Type": "text", "Access-Control-Allow-Origin": "*" }, "body": res_text } return response
from qcloud_cos import CosS3Client appid = os.environ.get('tencent_appid') secret_id = os.environ.get('tencent_secret_id') secret_key = os.environ.get('tencent_secret_key') region = os.environ.get('region') bucket = os.environ.get('cos_bucket').replace("-" + appid, "") print(os.environ) config = CosConfig( Secret_id=secret_id, Secret_key=secret_key, Region=region, ) client = CosS3Client(config) def getFileMd5(file_path): """ 获取文件md5值 :param file_path: 文件路径名 :return: 文件md5值 """ with open(file_path, 'rb') as f: md5obj = hashlib.md5() md5obj.update(f.read()) _hash = md5obj.hexdigest() return str(_hash).upper()
def __init__(self, region:str, secret_id:str, secret_key:str, bucket:str): config = CosConfig(Region=region, Secret_id=secret_id, Secret_key=secret_key) self.bucket = bucket self.client = CosS3Client(config)
def main_handler(event, context): logger.info("start main handler") if "Records" not in event.keys(): return {"errorMsg": "event is not come from cos"} secret_id = os.environ.get('TENCENTCLOUD_SECRETID') secret_key = os.environ.get('TENCENTCLOUD_SECRETKEY') token = os.environ.get('TENCENTCLOUD_SESSIONTOKEN') appid = event['Records'][0]['cos']['cosBucket']['appid'] config = CosConfig(Secret_id=secret_id, Secret_key=secret_key, Region=region, Token=token) cos_client = CosS3Client(config) bucket = event['Records'][0]['cos']['cosBucket']['name'] + '-' + str(appid) key = event['Records'][0]['cos']['cosObject']['key'] key = key.replace( '/' + str(appid) + '/' + event['Records'][0]['cos']['cosBucket']['name'] + '/', '', 1) download_path = '/tmp/{}'.format(key) tmpload_path = '/tmp/resized-{}'.format(key) logger.info("Key is " + key) logger.info("Get from [%s] to download file [%s]" % (bucket, key)) # download image from cos try: response = cos_client.get_object( Bucket=bucket, Key=key, ) response['Body'].get_stream_to_file(download_path) logger.info("Download file [%s] Success" % key) except CosServiceError as e: print(e.get_error_code()) print(e.get_error_msg()) print(e.get_resource_location()) return "Download File Fail" # detect idcard logger.info("Start Detection") CardSide = "FRONT" res_ai = idcard_detection(secret_id, secret_key, token, download_path, CardSide) if res_ai != -1: res_print = { "姓名:": res_ai["Name"], "性别:": res_ai["Sex"], "出生:": res_ai["Birth"], "住址:": res_ai["Address"], "民族:": res_ai["Nation"], "公民身份证号:": res_ai['IdNum'] } print(json.dumps(res_print).decode('unicode-escape')) else: CardSide = "BACK" res_ai = idcard_detection(secret_id, secret_key, token, download_path, CardSide) if res_ai != -1: res_print = { "有效期限:": res_ai["ValidDate"], "签发机关:": res_ai["Authority"] } print(json.dumps(res_print).decode('unicode-escape')) else: return "Detect Fail" # delete local file delete_local_file(str(download_path)) return res_print
def main_handler(event, context): logger.info("start main handler") if "Records" not in event.keys(): return {"errorMsg": "event is not come from cos"} print("Start Request {}", datetime.fromtimestamp(time()).strftime('%Y-%m-%d %H:%M:%S')) secret_id = os.environ.get('TENCENTCLOUD_SECRETID') # 使用环境变量中的 secretId secret_key = os.environ.get('TENCENTCLOUD_SECRETKEY') # 使用环境变量中的 secretKey token = os.environ.get('TENCENTCLOUD_SESSIONTOKEN') appid = event['Records'][0]['cos']['cosBucket']['appid'] config = CosConfig( Region=REGION, SecretId=secret_id, SecretKey=secret_key, Token=token, ) cos_client = CosS3Client(config) # Start downloading from COS bucket = event['Records'][0]['cos']['cosBucket']['name'] + '-' + str(appid) key = event['Records'][0]['cos']['cosObject']['key'] key = key.replace( '/' + str(appid) + '/' + event['Records'][0]['cos']['cosBucket']['name'] + '/', '', 1) download_path = '/tmp/{}'.format(key) print("Key is " + key) print("Get from [%s] to download file [%s]" % (bucket, key)) try: response = cos_client.get_object( Bucket=bucket, Key=key, ) response['Body'].get_stream_to_file(download_path) except CosServiceError as e: print(e.get_error_code()) print(e.get_error_msg()) print(e.get_resource_location()) return "Download log fail" logger.info("Download file [%s] Success" % key) print("Start analyzing data {}", datetime.fromtimestamp(time()).strftime('%Y-%m-%d %H:%M:%S')) urlList = {} statuelist = {} terminalList = {} timeList = {"24/May/2018 10:00-10:30": 0, "24/May/2018 10:30-11:00": 0} fileObject = open(download_path, 'rU') try: for line in fileObject: # Count URL URLstart = re.search("GET", line) URLend = re.search("mp4", line) if URLstart and URLend: url = line[URLstart.end() + 1:URLend.end()] if url in urlList: urlList[url] += 1 else: urlList[url] = 1 # Count Statue code Statuestart = re.search("HTTP/1.1", line) if Statuestart: StatueCode = line[Statuestart.end() + 2:Statuestart.end() + 5] if StatueCode in statuelist: statuelist[StatueCode] += 1 else: statuelist[StatueCode] = 1 # Count Terminal Device Terminalstart = re.search("\"-\"", line) TerminalEnd = re.search("\"-\" \"-\"", line) if Terminalstart and TerminalEnd: terminal = line[Terminalstart.end() + 2:TerminalEnd.start() - 2] if terminal in terminalList: terminalList[terminal] += 1 else: terminalList[terminal] = 1 # Count Timelist Timestarter = re.search("\[", line) if Timestarter: if int(line[Timestarter.end() + 15:Timestarter.end() + 17]) > 30: timeList["24/May/2018 10:30-11:00"] += 1 else: timeList["24/May/2018 10:00-10:30"] += 1 finally: fileObject.close() # Sort Result according to frequence URL_sorted_res = sorted(urlList.items(), key=operator.itemgetter(1), reverse=True) Statue_sorted_res = sorted(statuelist.items(), key=operator.itemgetter(1), reverse=True) Terminal_sorted_res = sorted(terminalList.items(), key=operator.itemgetter(1), reverse=True) Time_sorted_res = sorted(timeList.items(), key=operator.itemgetter(1), reverse=True) URLres = [] Statueres = [] Terminalres = [] Timeres = [] for i in range(3): URLres.append(URL_sorted_res[i]) Statueres.append(Statue_sorted_res[i]) Terminalres.append(Terminal_sorted_res[i]) for i in range(2): Timeres.append(Time_sorted_res[i]) print("Analyzing Successfully, Start writing to database {}", datetime.fromtimestamp(time()).strftime('%Y-%m-%d %H:%M:%S')) connection = pymysql.connect(host=DB_HOST, user=DB_USER, password=DB_USER_PASSWORD, port=int(DB_PORT), db=DB_NAME, charset='utf8', cursorclass=pymysql.cursors.DictCursor) try: with connection.cursor() as cursor: # Clean dirty data cursor.execute("DROP TABLE IF EXISTS url") cursor.execute("DROP TABLE IF EXISTS state") cursor.execute("DROP TABLE IF EXISTS terminal") cursor.execute("DROP TABLE IF EXISTS time") cursor.execute("CREATE TABLE url (URL TEXT NOT NULL, Count INT)") cursor.execute( "CREATE TABLE state (StateCode TEXT NOT NULL, Count INT)") cursor.execute( "CREATE TABLE terminal (Terminal TEXT NOT NULL, Count INT)") cursor.execute( "CREATE TABLE time (Timestatue TEXT NOT NULL, Count INT)") sql = "INSERT INTO `url` (`URL`, `Count`) VALUES (%s, %s)" for i in range(len(URLres)): cursor.execute(sql, (URLres[i][0], URLres[i][1])) sql = "INSERT INTO `state` (`StateCode`, `Count`) VALUES (%s, %s)" for i in range(len(Statueres)): cursor.execute(sql, (Statueres[i][0], Statueres[i][1])) sql = "INSERT INTO `terminal` (`Terminal`, `Count`) VALUES (%s, %s)" for i in range(len(Terminalres)): cursor.execute(sql, (Terminalres[i][0], Terminalres[i][1])) sql = "INSERT INTO `time` (`Timestatue`, `Count`) VALUES (%s, %s)" for i in range(len(Timeres)): cursor.execute(sql, (Timeres[i][0], Timeres[i][1])) connection.commit() finally: connection.close() print("Write to database successfully {}", datetime.fromtimestamp(time()).strftime('%Y-%m-%d %H:%M:%S')) return "LogAnalysis Success"
def main_handler(event, context): logger.info(json.dumps(event)) # 为了适配windows端用户 # 将ffmeg文件复制到/tmp下并赋予执行权限 subprocess.run('cp ./ffmpeg /tmp/ffmpeg && chmod 755 /tmp/ffmpeg', shell=True) region = os.environ.get('REGION') # 输出桶所在地域 dst_bucket = os.environ.get('DST_BUCKET') # 输出桶名称 dst_path = os.environ.get('DST_PATH') # 输出桶目录 dst_format = os.environ.get('DST_FORMAT') # 转码格式 cmd_origin = '/tmp/' + os.environ.get('FFMPEG_CMD') # ffmpeg命令 secret_id = os.environ.get('TENCENTCLOUD_SECRETID') secret_key = os.environ.get('TENCENTCLOUD_SECRETKEY') token = os.environ.get('TENCENTCLOUD_SESSIONTOKEN') ffmpeg_debug = os.environ.get('FFMPEG_DEBUG', 1) # 是否输出ffmpeg日志, 1为输出 0为不输出 read_limit = 1024 * 64 # 一次性读取多少输出数据(单位: byte),建议使用默认值即可,默认为64KB # 分块上传一次性最多上传多少数据(单位: byte),建议使用默认值即可,默认为10MB upload_size = 1024 * 1024 * 10 if "Records" in event.keys(): base_info = {} date = datetime.datetime.now() date_tag = date.strftime('%Y%m%d%H%M%S') cos_client = CosS3Client( CosConfig(Region=region, SecretId=secret_id, SecretKey=secret_key, Token=token)) key = "/".join( event['Records'][0]['cos']['cosObject']['key'].split("/")[3:]) # 生成输入文件的预签名 # cos sdk无token签名,因此这里需要手动拼接token download_input = cos_client.get_presigned_download_url( Bucket=dst_bucket, Key=key, Expired=60 * 60 * 12, ) download_input += '&x-cos-security-token={token}'.format(token=token) output_name = key.split('/')[-1].split('.')[0] + '.' + dst_format file_name, file_format = output_name.split('.') upload_name = '{file_name}-{date_tag}.{file_format}'.format( file_name=file_name, date_tag=date_tag, file_format=file_format) # 创建fifo fifo_path = '/tmp/fifo' + str(int(time.time())) os.mkfifo(fifo_path) cmd = cmd_origin.format(input=download_input, dst_format=dst_format, output=fifo_path) cmd_list = cmd.split(' ') # 日志上报与文件上传线程使用的一些公共参数 base_info['cos_client'] = cos_client base_info['region'] = region base_info['dst_format'] = dst_format base_info['dst_bucket'] = dst_bucket base_info['dst_path'] = dst_path base_info['ffmpeg_debug'] = ffmpeg_debug base_info['size'] = event['Records'][0]['cos']['cosObject']['size'] base_info['key'] = key base_info['upload_name'] = upload_name base_info['date_tag'] = date_tag base_info['fifo_path'] = fifo_path base_info['upload_size'] = upload_size base_info['read_limit'] = read_limit else: raise Exception('event does not come from COS') try: # 转码进程启动 proc = subprocess.Popen(cmd_list, stderr=subprocess.PIPE, universal_newlines=True) # 创建文件上传线程 task_list = [] cos_upload_task = threading.Thread(target=file_upload_ffmpeg_task, args=(proc, base_info)) # 创建日志输出线程 task_list.append(cos_upload_task) log_upload_task = threading.Thread(target=log_ffmpeg_task, args=(proc, base_info)) task_list.append(log_upload_task) # 启动线程 for task in task_list: task.start() # 等待线程结束 for task in task_list: task.join() except Exception as e: print(traceback.format_exc()) finally: # 显式删除fifo if os.path.exists(fifo_path): os.remove(fifo_path) # 显式删除/tmp下的ffmpeg可执行文件 if os.path.exists('/tmp/ffmpeg'): os.remove('/tmp/ffmpeg') return {'code': 200, 'Msg': 'success'}