def main_handler(event, context):
logger.info("start main handler")
if "Records" not in event.keys():
return {"errorMsg": "event is not come from cos"}
#初始化COS及获取COS文件信息
appid = event['Records'][0]['cos']['cosBucket']['appid']
bucket = event['Records'][0]['cos']['cosBucket']['name'] + '-' + str(appid)
key = event['Records'][0]['cos']['cosObject']['key']
key = key.replace('/' + str(appid) + '/' + event['Records'][0]['cos']['cosBucket']['name'] + '/', '', 1)
if key[-1] == '/':
return {"errorMsg": "this is floder"}
# 提取文件名 shotname
folder_path,file_name = os.path.split(key)
download_path = '/tmp/{}'.format(file_name)
logger.info("File_name is " + key)
logger.info("download_path is " + download_path)
logger.info("Get from [%s] to download file [%s]" % (bucket, key))
#使用临时秘钥
secret_id = os.environ.get('TENCENTCLOUD_SECRETID')
secret_key = os.environ.get('TENCENTCLOUD_SECRETKEY')
token = os.environ.get('TENCENTCLOUD_SESSIONTOKEN')
# 从COS下载需要同步的文件
cos_client_source = cosClient = CosS3Client(CosConfig(Region=source_region, SecretId=secret_id, SecretKey=secret_key,Token=token))
try:
response = cos_client_source.get_object(Bucket=bucket, Key=key, )
response['Body'].get_stream_to_file(download_path)
logger.info("Download file [%s] Success" % key)
except CosServiceError as e:
logger.info("Download file [%s] Fail" % key)
logger.error(e.get_error_code())
logger.error(e.get_error_msg())
logger.error(e.get_resource_location())
delete_local_file(str(download_path))
return ("Download file [%s] from [%s] fail" % (key,bucket))
# 同步文件到异地COS bucket
logger.info("Start to upload file to target_bucket")
cos_client_target = cosClient = CosS3Client(CosConfig(Region=target_region, SecretId=target_secret_id, SecretKey=target_secret_key,))
try:
response = cos_client_target.put_object_from_local_file(
Bucket=target_bucket,
LocalFilePath=download_path,
Key=key)
logger.info("Upload result is [%s]" % response)
except CosServiceError as e:
logger.error(e.get_error_code())
logger.error(e.get_error_msg())
logger.error(e.get_resource_location())
delete_local_file(str(download_path))
return ("Upload file [%s] from [%s] fail" % (key,bucket))
delete_local_file(str(download_path))
return "Sync File Success"
def main_handler(event, context):
logger.info("start unpack template function")
secret_id = os.getenv('TENCENTCLOUD_SECRETID')
secret_key = os.getenv('TENCENTCLOUD_SECRETKEY')
token = os.getenv('TENCENTCLOUD_SESSIONTOKEN')
config = CosConfig(Secret_id=secret_id, Secret_key=secret_key, Region=region, Token=token)
client = CosS3Client(config)
for record in event['Records']:
try:
appid = record['cos']['cosBucket']['appid']
bucket = record['cos']['cosBucket']['name'] + '-' + appid
filename = os.path.basename(record['cos']['cosObject']['url'])
download_path = '/tmp/{}'.format(filename.encode('gb18030'))
key = record['cos']['cosObject']['key']
key = key.replace('/' + appid + '/' + record['cos']['cosBucket']['name'] + '/', '', 1)
# 创建本地解压路径
isExists = os.path.exists('/tmp/unpack')
if not isExists:
os.mkdir('/tmp/unpack')
unpack_path = '/tmp/unpack'
# 提取文件名 shotname
(filepath, tempfilename) = os.path.split(filename);
(shotname, extension) = os.path.splitext(tempfilename);
if extension[1:] not in unpack_suffix.split(','):
logger.info("object suffix is [%s], expected: [%s]", extension, unpack_suffix)
return "object suffix is [%s], expected: [%s]" % (extension, unpack_suffix)
logger.info("object name is [%s]", shotname)
# download rar from cos
logger.info("get from [%s] to download object [%s]", bucket, filename)
try:
response = client.get_object(Bucket=bucket, Key=key, )
response['Body'].get_stream_to_file(download_path)
logger.info("download object [%s] Success", filename)
except CosServiceError as e:
print(e.get_error_code())
print(e.get_error_msg())
print(e.get_resource_location())
logger.info("download object [%s] failed", filename)
return "download object fail"
# start to extract archive file and upload to bucket_upload
logger.info("start to extract archive file")
Archive(download_path).extractall(unpack_path, auto_create_dir=True)
logger.info("extract success")
upload_local_file(client, '/tmp/unpack', target_dir)
# clean files
delete_local_file(str(download_path))
delete_local_file(str(unpack_path))
return "extract and upload success"
except Exception as e:
print(e)
raise e
return "extract and upload fail"
def main_handler(event, context):
logger.info("start main handler")
if "Records" not in event.keys():
return {"errorMsg": "event is not come from cos"}
secret_id = os.environ.get('TENCENTCLOUD_SECRETID') # 使用环境变量中的 secretId
secret_key = os.environ.get('TENCENTCLOUD_SECRETKEY') # 使用环境变量中的 secretKey
token = os.environ.get('TENCENTCLOUD_SESSIONTOKEN')
config = CosConfig(
Region=region,
SecretId=secret_id,
SecretKey=secret_key,
Token=token,
)
cos_client = CosS3Client(config)
start_time = datetime.datetime.now()
res = map_caller(event, context, cos_client)
end_time = datetime.datetime.now()
print("data mapping duration: " +
str((end_time - start_time).microseconds / 1000) + "ms")
if res == 0:
return "Data mapping SUCCESS"
else:
return "Data mapping FAILED"
def __init__(self,
kafka_instance_id,
topic_name,
topic_id,
kafka_address,
bucket_address,
partition_max_to_cos_bytes,
partition_max_timeout_ms,
partition_id,
consumer_timeout_ms,
group_id,
offset_type,
region,
secret_id,
secret_key,
token=None):
self.topic_name = topic_name
self.topic_id = topic_id
self.kafka_address = kafka_address
self.bucket_address = bucket_address
self.kafka_instance_id = kafka_instance_id
self.partition_max_to_cos_bytes = partition_max_to_cos_bytes
self.partition_max_timeout_ms = partition_max_timeout_ms
self.partition_id = partition_id
self.consumer_timeout_ms = consumer_timeout_ms
self.group_id = group_id
self.offset_type = offset_type
config = CosConfig(Region=region,
SecretId=secret_id,
SecretKey=secret_key,
Token=token) # 获取配置对象
self.client = CosS3Client(config)
def getPresignedUrl(event, context):
print('event', event)
print('context', context)
body = json.loads(event['body'])
# 可以通过客户端传来的token进行鉴权，只有鉴权通过才可以获得临时上传地址
# 这一部分可以按需修改，例如用户的token可以在redis获取，可以通过某些加密方法获取等
# 也可以是传来一个username和一个token，然后去数据库中找这个username对应的token是否
# 与之匹配等，这样会尽可能的提升安全性
if "key" not in body or "token" not in body or body['token'] != 'mytoken' or "key" not in body:
return {"url": None}
# 初始化COS对象
region = os.environ.get("region")
secret_id = os.environ.get("TENCENTCLOUD_SECRETID")
secret_key = os.environ.get("TENCENTCLOUD_SECRETKEY")
token = os.environ.get("TENCENTCLOUD_SESSIONTOKEN")
config = CosConfig(Region=region, SecretId=secret_id, SecretKey=secret_key, Token=token)
client = CosS3Client(config)
response = client.get_presigned_url(
Method='PUT',
Bucket=os.environ.get('bucket_name'),
Key=body['key'],
Expired=30,
)
return {"url": response.split("?sign=")[0],
"sign": urllib.parse.unquote(response.split("?sign=")[1]),
"token": os.environ.get("TENCENTCLOUD_SESSIONTOKEN")}
def uploadToScf(event, context):
print('event', event)
print('context', context)
body = json.loads(event['body'])
# 可以通过客户端传来的token进行鉴权，只有鉴权通过才可以获得临时上传地址
# 这一部分可以按需修改，例如用户的token可以在redis获取，可以通过某些加密方法获取等
# 也可以是传来一个username和一个token，然后去数据库中找这个username对应的token是否
# 与之匹配等，这样会尽可能的提升安全性
if "key" not in body or "token" not in body or body['token'] != 'mytoken' or "key" not in body:
return {"url": None}
pictureBase64 = body["picture"].split("base64,")[1]
with open('/tmp/%s' % body['key'], 'wb') as f:
f.write(base64.b64decode(pictureBase64))
region = os.environ.get("region")
secret_id = os.environ.get("TENCENTCLOUD_SECRETID")
secret_key = os.environ.get("TENCENTCLOUD_SECRETKEY")
token = os.environ.get("TENCENTCLOUD_SESSIONTOKEN")
config = CosConfig(Region=region, SecretId=secret_id, SecretKey=secret_key, Token=token)
client = CosS3Client(config)
response = client.upload_file(
Bucket=os.environ.get("bucket_name"),
LocalFilePath='/tmp/%s' % body['key'],
Key=body['key'],
)
return {
"uploaded": 1,
"url": 'https://%s.cos.%s.myqcloud.com' % (
os.environ.get("bucket_name"), os.environ.get("region")) + body['key']
}
def main_handler(event, context):
secret_id = os.environ.get('TENCENTCLOUD_SECRETID') # Using the secterId in environment variables. 使用环境变量中的 secretId
secret_key = os.environ.get('TENCENTCLOUD_SECRETKEY') # Using the secretKey in environment variables. 使用环境变量中的 secretKey
token = os.environ.get('TENCENTCLOUD_SESSIONTOKEN')
config = CosConfig(Region=REGION, SecretId=secret_id, SecretKey=secret_key, Token=token,)
cos_client = CosS3Client(config)
connection = pymysql.connect(host=DB_HOST,
user=DB_USER,
password=DB_USER_PASSWORD,
port=int(DB_PORT),
db=DB_NAME,
charset='utf8',
cursorclass=pymysql.cursors.DictCursor)
with connection.cursor() as cursor:
sql = 'show databases'
cursor.execute(sql)
res = cursor.fetchall()
print res
timestr = time.strftime('%Y%m%d-%H%M%S', time.localtime(time.time()))
filename = DB_NAME+"-"+timestr+".sql"
filepath = BACKUP_PATH+os.sep+filename
print ("Start Backup")
dumpcmd = "./mysqldump -h" + DB_HOST + " -P" + DB_PORT + " -u" + DB_USER + " -p" + DB_USER_PASSWORD + " " + DB_NAME + " > " + filepath
print (dumpcmd)
print os.popen(dumpcmd).read()
print ("Backup script completed")
print ("Your backups has been created in '" + filepath + "' file")
print (os.popen('ls -l /tmp').read())
print ("finish backup")
print ("start send to cos")
backup2cos(cos_client,filepath,BACKUP_BUCKET,"/"+filename)
print ("finish send to cos")
def __init__(self, *args, **kwargs):
appid = int(kwargs['appid'])
region = kwargs['region']
accesskeyid = unicode(kwargs['accesskeyid'])
accesskeysecret = unicode(kwargs['accesskeysecret'])
bucket = unicode(kwargs['bucket'])
if 'prefix_dir' in kwargs:
self._prefix_dir = kwargs['prefix_dir']
else:
self._prefix_dir = None
if 'part_size' in kwargs:
self._part_size = kwargs['part_size']
else:
self._part_size = 1
conf = CosConfig(Appid=appid,
Access_id=str(accesskeyid),
Access_key=str(accesskeysecret),
Region=region)
self._cos_client = CosS3Client(conf=conf, retry=1)
self._bucket = bucket
self._max_retry = 20
self._appid = appid
self._region = region
self._accesskeyid = str(accesskeyid)
self._accesskeysecret = str(accesskeysecret)
def write_to_cos(path, cos_path):
"""输出用户信息"""
matchObj = re.match(r'https://(.*).cos.(.*).myqcloud.com(.*)', cos_path,
re.M | re.I)
dst_bucket = matchObj.group(1)
dst_path = matchObj.group(3)
region = matchObj.group(2)
secret_id = os.environ.get('TENCENTCLOUD_SECRETID')
secret_key = os.environ.get('TENCENTCLOUD_SECRETKEY')
token = os.environ.get('TENCENTCLOUD_SESSIONTOKEN')
cos_client = CosS3Client(
CosConfig(Region=region,
SecretId=secret_id,
SecretKey=secret_key,
Token=token))
cos_client.put_object_from_local_file(Bucket=dst_bucket,
LocalFilePath=path,
Key=dst_path + '/' +
path.split('/')[-1])
delete_local_file(path)
return {"file": dst_bucket + dst_path + '/' + path.split('/')[-1]}
def main_handler(event, context):
logger.info("start main handler")
if "Records" not in event.keys():
return {"errorMsg": "event is not come from cos"}
secret_id = os.environ.get(
'TENCENT_SECRET_ID'
) # Using the secterId in environment variables. 使用环境变量中的 secretId
secret_key = os.environ.get(
'TENCENT_SECRET_KEY'
) # Using the secretKey in environment variables. 使用环境变量中的 secretKey
config = CosConfig(
Region=region,
SecretId=secret_id,
SecretKey=secret_key,
)
cos_client = CosS3Client(config)
start_time = datetime.datetime.now()
res = reduce_caller(event, context, cos_client)
end_time = datetime.datetime.now()
print("data reducing duration: " +
str((end_time - start_time).microseconds / 1000) + "ms")
if res == 0:
return "Data reducing SUCCESS"
else:
return "Data reducing FAILED"
def __init__(self, config):
cos_config = CosConfig(
Region=config['cos_region'],
Secret_id=config['secret_id'],
Secret_key=config['secret_key'],
Token=config.get('token', None),
)
self.cos_client = CosS3Client(cos_config)
self.cos_bucket = config['cos_bucket']
self.cos_key = config['cos_key']
self.url = config['url']
def main_handler(event, context):
logger.info("start main handler")
#使用临时秘钥初始化COS Client
secret_id = os.environ.get('TENCENTCLOUD_SECRETID')
secret_key = os.environ.get('TENCENTCLOUD_SECRETKEY')
token = os.environ.get('TENCENTCLOUD_SESSIONTOKEN')
cosClient = CosS3Client(
CosConfig(Region=region,
SecretId=secret_id,
SecretKey=secret_key,
Token=token))
#判断请求是否从API网关传递,通过网关传递源文件下载地址，在body中获取
if "body" in event.keys():
download_path = event['body']
key = download_path.split('/')[-1]
logger.info('api download_path: ' + download_path)
#判断请求是否从COS传递
elif "Records" in event.keys():
#从event里获取COS文件信息，并获取下载地址
bucket = event['Records'][0]['cos']['cosBucket']['name'] + '-' + event[
'Records'][0]['cos']['cosBucket']['appid']
key = "/".join(
event['Records'][0]['cos']['cosObject']['key'].split("/")[3:])
download_path = event['Records'][0]['cos']['cosObject']['url']
logger.info('cos download_path: ' + download_path)
else:
return {
"code": 410,
"errorMsg": "event does not come from COS or APIGW"
}
logger.info('key: ' + key)
upload_path = '/tmp/new-' + key.split('/')[-1]
logger.info('upload_path: ' + upload_path)
# 执行ffmpeg命令，从下载地址读流的方式进行转码
child = subprocess.run(video_press % (download_path, upload_path),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
close_fds=True,
shell=True)
# 上传视频，可自定义上传路径
cosClient.put_object_from_local_file(Bucket=target_bucket,
LocalFilePath=upload_path,
Key=target_path + '/' +
upload_path.split('/')[-1])
delete_local_file(upload_path)
return 'success'
def __init__(self,
start_time,
request_id,
kafka_instance_id,
topic_name,
topic_id,
kafka_address,
bucket_address,
file_path_prefix,
once_max_to_cos_bytes,
function_timeout_ms,
partition_id,
group_id,
offset_type,
region,
secret_id,
secret_key,
token=None):
self.start_time = start_time
self.request_id = request_id
self.kafka_instance_id = kafka_instance_id
self.topic_name = topic_name
self.topic_id = topic_id
self.kafka_address = kafka_address
self.bucket_address = bucket_address
self.file_path_prefix = file_path_prefix
self.once_max_to_cos_bytes = once_max_to_cos_bytes
self.function_timeout_ms = function_timeout_ms
self.partition_id = partition_id
self.group_id = group_id
self.offset_type = offset_type
config = CosConfig(Region=region,
SecretId=secret_id,
SecretKey=secret_key,
Token=token) # 获取配置对象
self.cos_client = CosS3Client(config)
self.cos_upload_speed = float(init_cos_upload_speed)
self.start_offset = PyKafkaOffsetType.EARLIEST
def main_handler(event, context):
logger.info("start main handler")
if "Records" not in event.keys():
return {"errorMsg": "event is not come from cos"}
logger.info("source_bucket is " + source_bucket)
logger.info("target_bucket is " + target_bucket)
#使用临时秘钥
secret_id = os.environ.get('TENCENTCLOUD_SECRETID')
secret_key = os.environ.get('TENCENTCLOUD_SECRETKEY')
token = os.environ.get('TENCENTCLOUD_SESSIONTOKEN')
cosClient = CosS3Client(
CosConfig(Region=region,
SecretId=secret_id,
SecretKey=secret_key,
Token=token))
appid = event['Records'][0]['cos']['cosBucket']['appid']
key = event['Records'][0]['cos']['cosObject']['key']
key = key.replace(
'/' + str(appid) + '/' +
event['Records'][0]['cos']['cosBucket']['name'] + '/', '', 1)
if key[-1] == '/':
return {"errorMsg": "this is floder,do not need to sync"}
logger.info("key:" + key)
response = cosClient.copy_object(Bucket=target_bucket,
Key=key,
CopySource={
'Bucket': source_bucket,
'Key': key,
'Region': region
})
print("response is ", response)
return 'copy success'
def __init__(self, config):
self.config = config
cred = credential.Credential(config['secret_id'], config['secret_key'],
config['token'])
httpProfile = HttpProfile()
httpProfile.endpoint = "cdn.tencentcloudapi.com"
clientProfile = ClientProfile()
clientProfile.httpProfile = httpProfile
cdnClient = cdn_client.CdnClient(cred, config['scf_region'],
clientProfile)
self.cdn_client = cdnClient
self.cos_path = config['cos_path']
cos_config = CosConfig(
Region=config['cos_region'],
Secret_id=config['secret_id'],
Secret_key=config['secret_key'],
Token=config.get('token', None),
)
self.cos_client = CosS3Client(cos_config)
self.cos_bucket = config['cos_bucket']
appid = os.environ.get('appid') # 请在函数配置中添加环境变量appid，并填入您的 APPID
secret_id = os.environ.get(
'secret_id') # 请在函数配置中添加环境变量secret_id，并填入您的 SecretId
secret_key = os.environ.get(
'secret_key') # 请在函数配置中添加环境变量secret_key，并填入您的 SecretKey
bucket_region = os.environ.get(
'bucket_region') # 请在函数配置中添加环境变量bucket_region，并填入您COS Bucket所在的地域
token = ''
bucket_upload = os.environ.get(
'bucket_upload') # 请在函数配置中添加环境变量bucket_upload，并填入您要用来存放图片的Bucket名
config = CosConfig(Secret_id=secret_id,
Secret_key=secret_key,
Region=bucket_region,
Token=token)
client_cos = CosS3Client(config)
def delete_local_file(src):
logger.info("delete files and folders")
if os.path.isfile(src):
try:
os.remove(src)
except:
pass
elif os.path.isdir(src):
for item in os.listdir(src):
itemsrc = os.path.join(src, item)
delete_file_folder(itemsrc)
try:
os.rmdir(src)
from qcloud_cos_v5 import CosConfig
from qcloud_cos_v5 import CosS3Client
from tencentcloud.common import credential
from tbp import tbp_client, models as tbp_models
from tts import tts_client, models as tts_models
from tencentcloud.scf.v20180416 import scf_client, models as scf_models
bot_id = os.environ.get('bot_id')
bucket = os.environ.get('bucket')
secret_id = os.environ.get('secret_id')
secret_key = os.environ.get('secret_key')
region = os.environ.get('region')
wxtoken = os.environ.get('wxtoken')
appid = os.environ.get('appid')
secret = os.environ.get('secret')
cosClient = CosS3Client(
CosConfig(Region=region, SecretId=secret_id, SecretKey=secret_key))
scfClient = scf_client.ScfClient(credential.Credential(secret_id, secret_key),
region)
tbpClient = tbp_client.TbpClient(credential.Credential(secret_id, secret_key),
region)
ttsClient = tts_client.TtsClient(credential.Credential(secret_id, secret_key),
region)
key = 'news/content.json'
indexKey = 'news/content_index.json'
accessTokenKey = 'access/token.json'
accessToken = None
articlesList = None
def getAccessToken():
def main_handler(event, context):
logger.info("start main handler")
if "requestContext" not in event.keys():
return {"code": 410, "errorMsg": "event is not come from api gateway"}
if "body" not in event.keys():
return {
"isBase64Encoded": False,
"statusCode": 200,
"headers": {
"Content-Type": "text",
"Access-Control-Allow-Origin": "*"
},
"body": "there is no file from api gateway"
}
# save api gateway file to local temp file
logger.info("Start to download images from APIGW")
time = datetime.datetime.now()
file_name = '{}'.format(time) + "-test.jpg"
logger.info("file_name is : %s" % file_name)
local_path = u'/tmp/{}'.format(file_name)
logger.info("local_path is : %s" % local_path)
with open(local_path, 'w') as wfile:
wfile.write(base64.b64decode(event['body']))
# start to upload to cos
secret_id = os.environ.get('TENCENTCLOUD_SECRETID') # 使用环境变量中的 secretId
secret_key = os.environ.get('TENCENTCLOUD_SECRETKEY') # 使用环境变量中的 secretKey
token = os.environ.get('TENCENTCLOUD_SESSIONTOKEN')
config = CosConfig(
Region=bucket_region,
SecretId=secret_id,
SecretKey=secret_key,
Token=token,
)
cos_client = CosS3Client(config)
logger.info("Start to upload images to cos")
res_cos = cos_client.put_object_from_local_file(Bucket=bucket_upload,
LocalFilePath=local_path,
Key='{}'.format(file_name))
logger.info("upload to cos result is : %s" % res_cos)
# start to detection
logger.info("Start to detection")
try:
cred = credential.Credential(secret_id, secret_key, token)
httpProfile = HttpProfile()
httpProfile.endpoint = "ocr.tencentcloudapi.com"
clientProfile = ClientProfile()
clientProfile.httpProfile = httpProfile
client = ocr_client.OcrClient(cred, "ap-beijing", clientProfile)
req = models.GeneralBasicOCRRequest()
params = '{"ImageBase64":"%s"}' % event['body']
req.from_json_string(params)
resp = client.GeneralBasicOCR(req)
res_ai = json.loads(resp.to_json_string())
res_text = " "
print len(res_ai["TextDetections"])
for i in range(len(res_ai["TextDetections"])):
res_text = res_text + str(
res_ai["TextDetections"][i]["DetectedText"].encode('utf-8'))
except TencentCloudSDKException as err:
print(err)
delete_local_file(local_path)
response = {
"isBase64Encoded": False,
"statusCode": 200,
"headers": {
"Content-Type": "text",
"Access-Control-Allow-Origin": "*"
},
"body": res_text
}
return response
from qcloud_cos import CosS3Client
appid = os.environ.get('tencent_appid')
secret_id = os.environ.get('tencent_secret_id')
secret_key = os.environ.get('tencent_secret_key')
region = os.environ.get('region')
bucket = os.environ.get('cos_bucket').replace("-" + appid, "")
print(os.environ)
config = CosConfig(
Secret_id=secret_id,
Secret_key=secret_key,
Region=region,
)
client = CosS3Client(config)
def getFileMd5(file_path):
"""
获取文件md5值
:param file_path: 文件路径名
:return: 文件md5值
"""
with open(file_path, 'rb') as f:
md5obj = hashlib.md5()
md5obj.update(f.read())
_hash = md5obj.hexdigest()
return str(_hash).upper()
def __init__(self, region:str, secret_id:str, secret_key:str, bucket:str):
config = CosConfig(Region=region, Secret_id=secret_id, Secret_key=secret_key)
self.bucket = bucket
self.client = CosS3Client(config)
def main_handler(event, context):
logger.info("start main handler")
if "Records" not in event.keys():
return {"errorMsg": "event is not come from cos"}
secret_id = os.environ.get('TENCENTCLOUD_SECRETID')
secret_key = os.environ.get('TENCENTCLOUD_SECRETKEY')
token = os.environ.get('TENCENTCLOUD_SESSIONTOKEN')
appid = event['Records'][0]['cos']['cosBucket']['appid']
config = CosConfig(Secret_id=secret_id,
Secret_key=secret_key,
Region=region,
Token=token)
cos_client = CosS3Client(config)
bucket = event['Records'][0]['cos']['cosBucket']['name'] + '-' + str(appid)
key = event['Records'][0]['cos']['cosObject']['key']
key = key.replace(
'/' + str(appid) + '/' +
event['Records'][0]['cos']['cosBucket']['name'] + '/', '', 1)
download_path = '/tmp/{}'.format(key)
tmpload_path = '/tmp/resized-{}'.format(key)
logger.info("Key is " + key)
logger.info("Get from [%s] to download file [%s]" % (bucket, key))
# download image from cos
try:
response = cos_client.get_object(
Bucket=bucket,
Key=key,
)
response['Body'].get_stream_to_file(download_path)
logger.info("Download file [%s] Success" % key)
except CosServiceError as e:
print(e.get_error_code())
print(e.get_error_msg())
print(e.get_resource_location())
return "Download File Fail"
# detect idcard
logger.info("Start Detection")
CardSide = "FRONT"
res_ai = idcard_detection(secret_id, secret_key, token, download_path,
CardSide)
if res_ai != -1:
res_print = {
"姓名：": res_ai["Name"],
"性别：": res_ai["Sex"],
"出生：": res_ai["Birth"],
"住址：": res_ai["Address"],
"民族：": res_ai["Nation"],
"公民身份证号：": res_ai['IdNum']
}
print(json.dumps(res_print).decode('unicode-escape'))
else:
CardSide = "BACK"
res_ai = idcard_detection(secret_id, secret_key, token, download_path,
CardSide)
if res_ai != -1:
res_print = {
"有效期限：": res_ai["ValidDate"],
"签发机关：": res_ai["Authority"]
}
print(json.dumps(res_print).decode('unicode-escape'))
else:
return "Detect Fail"
# delete local file
delete_local_file(str(download_path))
return res_print
def main_handler(event, context):
logger.info("start main handler")
if "Records" not in event.keys():
return {"errorMsg": "event is not come from cos"}
print("Start Request {}",
datetime.fromtimestamp(time()).strftime('%Y-%m-%d %H:%M:%S'))
secret_id = os.environ.get('TENCENTCLOUD_SECRETID') # 使用环境变量中的 secretId
secret_key = os.environ.get('TENCENTCLOUD_SECRETKEY') # 使用环境变量中的 secretKey
token = os.environ.get('TENCENTCLOUD_SESSIONTOKEN')
appid = event['Records'][0]['cos']['cosBucket']['appid']
config = CosConfig(
Region=REGION,
SecretId=secret_id,
SecretKey=secret_key,
Token=token,
)
cos_client = CosS3Client(config)
# Start downloading from COS
bucket = event['Records'][0]['cos']['cosBucket']['name'] + '-' + str(appid)
key = event['Records'][0]['cos']['cosObject']['key']
key = key.replace(
'/' + str(appid) + '/' +
event['Records'][0]['cos']['cosBucket']['name'] + '/', '', 1)
download_path = '/tmp/{}'.format(key)
print("Key is " + key)
print("Get from [%s] to download file [%s]" % (bucket, key))
try:
response = cos_client.get_object(
Bucket=bucket,
Key=key,
)
response['Body'].get_stream_to_file(download_path)
except CosServiceError as e:
print(e.get_error_code())
print(e.get_error_msg())
print(e.get_resource_location())
return "Download log fail"
logger.info("Download file [%s] Success" % key)
print("Start analyzing data {}",
datetime.fromtimestamp(time()).strftime('%Y-%m-%d %H:%M:%S'))
urlList = {}
statuelist = {}
terminalList = {}
timeList = {"24/May/2018 10:00-10:30": 0, "24/May/2018 10:30-11:00": 0}
fileObject = open(download_path, 'rU')
try:
for line in fileObject:
# Count URL
URLstart = re.search("GET", line)
URLend = re.search("mp4", line)
if URLstart and URLend:
url = line[URLstart.end() + 1:URLend.end()]
if url in urlList:
urlList[url] += 1
else:
urlList[url] = 1
# Count Statue code
Statuestart = re.search("HTTP/1.1", line)
if Statuestart:
StatueCode = line[Statuestart.end() + 2:Statuestart.end() + 5]
if StatueCode in statuelist:
statuelist[StatueCode] += 1
else:
statuelist[StatueCode] = 1
# Count Terminal Device
Terminalstart = re.search("\"-\"", line)
TerminalEnd = re.search("\"-\" \"-\"", line)
if Terminalstart and TerminalEnd:
terminal = line[Terminalstart.end() + 2:TerminalEnd.start() -
2]
if terminal in terminalList:
terminalList[terminal] += 1
else:
terminalList[terminal] = 1
# Count Timelist
Timestarter = re.search("\[", line)
if Timestarter:
if int(line[Timestarter.end() + 15:Timestarter.end() +
17]) > 30:
timeList["24/May/2018 10:30-11:00"] += 1
else:
timeList["24/May/2018 10:00-10:30"] += 1
finally:
fileObject.close()
# Sort Result according to frequence
URL_sorted_res = sorted(urlList.items(),
key=operator.itemgetter(1),
reverse=True)
Statue_sorted_res = sorted(statuelist.items(),
key=operator.itemgetter(1),
reverse=True)
Terminal_sorted_res = sorted(terminalList.items(),
key=operator.itemgetter(1),
reverse=True)
Time_sorted_res = sorted(timeList.items(),
key=operator.itemgetter(1),
reverse=True)
URLres = []
Statueres = []
Terminalres = []
Timeres = []
for i in range(3):
URLres.append(URL_sorted_res[i])
Statueres.append(Statue_sorted_res[i])
Terminalres.append(Terminal_sorted_res[i])
for i in range(2):
Timeres.append(Time_sorted_res[i])
print("Analyzing Successfully, Start writing to database {}",
datetime.fromtimestamp(time()).strftime('%Y-%m-%d %H:%M:%S'))
connection = pymysql.connect(host=DB_HOST,
user=DB_USER,
password=DB_USER_PASSWORD,
port=int(DB_PORT),
db=DB_NAME,
charset='utf8',
cursorclass=pymysql.cursors.DictCursor)
try:
with connection.cursor() as cursor:
# Clean dirty data
cursor.execute("DROP TABLE IF EXISTS url")
cursor.execute("DROP TABLE IF EXISTS state")
cursor.execute("DROP TABLE IF EXISTS terminal")
cursor.execute("DROP TABLE IF EXISTS time")
cursor.execute("CREATE TABLE url (URL TEXT NOT NULL, Count INT)")
cursor.execute(
"CREATE TABLE state (StateCode TEXT NOT NULL, Count INT)")
cursor.execute(
"CREATE TABLE terminal (Terminal TEXT NOT NULL, Count INT)")
cursor.execute(
"CREATE TABLE time (Timestatue TEXT NOT NULL, Count INT)")
sql = "INSERT INTO `url` (`URL`, `Count`) VALUES (%s, %s)"
for i in range(len(URLres)):
cursor.execute(sql, (URLres[i][0], URLres[i][1]))
sql = "INSERT INTO `state` (`StateCode`, `Count`) VALUES (%s, %s)"
for i in range(len(Statueres)):
cursor.execute(sql, (Statueres[i][0], Statueres[i][1]))
sql = "INSERT INTO `terminal` (`Terminal`, `Count`) VALUES (%s, %s)"
for i in range(len(Terminalres)):
cursor.execute(sql, (Terminalres[i][0], Terminalres[i][1]))
sql = "INSERT INTO `time` (`Timestatue`, `Count`) VALUES (%s, %s)"
for i in range(len(Timeres)):
cursor.execute(sql, (Timeres[i][0], Timeres[i][1]))
connection.commit()
finally:
connection.close()
print("Write to database successfully {}",
datetime.fromtimestamp(time()).strftime('%Y-%m-%d %H:%M:%S'))
return "LogAnalysis Success"
def main_handler(event, context):
logger.info(json.dumps(event))
# 为了适配windows端用户
# 将ffmeg文件复制到/tmp下并赋予执行权限
subprocess.run('cp ./ffmpeg /tmp/ffmpeg && chmod 755 /tmp/ffmpeg',
shell=True)
region = os.environ.get('REGION') # 输出桶所在地域
dst_bucket = os.environ.get('DST_BUCKET') # 输出桶名称
dst_path = os.environ.get('DST_PATH') # 输出桶目录
dst_format = os.environ.get('DST_FORMAT') # 转码格式
cmd_origin = '/tmp/' + os.environ.get('FFMPEG_CMD') # ffmpeg命令
secret_id = os.environ.get('TENCENTCLOUD_SECRETID')
secret_key = os.environ.get('TENCENTCLOUD_SECRETKEY')
token = os.environ.get('TENCENTCLOUD_SESSIONTOKEN')
ffmpeg_debug = os.environ.get('FFMPEG_DEBUG',
1) # 是否输出ffmpeg日志, 1为输出 0为不输出
read_limit = 1024 * 64 # 一次性读取多少输出数据(单位: byte),建议使用默认值即可,默认为64KB
# 分块上传一次性最多上传多少数据(单位: byte),建议使用默认值即可,默认为10MB
upload_size = 1024 * 1024 * 10
if "Records" in event.keys():
base_info = {}
date = datetime.datetime.now()
date_tag = date.strftime('%Y%m%d%H%M%S')
cos_client = CosS3Client(
CosConfig(Region=region,
SecretId=secret_id,
SecretKey=secret_key,
Token=token))
key = "/".join(
event['Records'][0]['cos']['cosObject']['key'].split("/")[3:])
# 生成输入文件的预签名
# cos sdk无token签名，因此这里需要手动拼接token
download_input = cos_client.get_presigned_download_url(
Bucket=dst_bucket,
Key=key,
Expired=60 * 60 * 12,
)
download_input += '&x-cos-security-token={token}'.format(token=token)
output_name = key.split('/')[-1].split('.')[0] + '.' + dst_format
file_name, file_format = output_name.split('.')
upload_name = '{file_name}-{date_tag}.{file_format}'.format(
file_name=file_name, date_tag=date_tag, file_format=file_format)
# 创建fifo
fifo_path = '/tmp/fifo' + str(int(time.time()))
os.mkfifo(fifo_path)
cmd = cmd_origin.format(input=download_input,
dst_format=dst_format,
output=fifo_path)
cmd_list = cmd.split(' ')
# 日志上报与文件上传线程使用的一些公共参数
base_info['cos_client'] = cos_client
base_info['region'] = region
base_info['dst_format'] = dst_format
base_info['dst_bucket'] = dst_bucket
base_info['dst_path'] = dst_path
base_info['ffmpeg_debug'] = ffmpeg_debug
base_info['size'] = event['Records'][0]['cos']['cosObject']['size']
base_info['key'] = key
base_info['upload_name'] = upload_name
base_info['date_tag'] = date_tag
base_info['fifo_path'] = fifo_path
base_info['upload_size'] = upload_size
base_info['read_limit'] = read_limit
else:
raise Exception('event does not come from COS')
try:
# 转码进程启动
proc = subprocess.Popen(cmd_list,
stderr=subprocess.PIPE,
universal_newlines=True)
# 创建文件上传线程
task_list = []
cos_upload_task = threading.Thread(target=file_upload_ffmpeg_task,
args=(proc, base_info))
# 创建日志输出线程
task_list.append(cos_upload_task)
log_upload_task = threading.Thread(target=log_ffmpeg_task,
args=(proc, base_info))
task_list.append(log_upload_task)
# 启动线程
for task in task_list:
task.start()
# 等待线程结束
for task in task_list:
task.join()
except Exception as e:
print(traceback.format_exc())
finally:
# 显式删除fifo
if os.path.exists(fifo_path):
os.remove(fifo_path)
# 显式删除/tmp下的ffmpeg可执行文件
if os.path.exists('/tmp/ffmpeg'):
os.remove('/tmp/ffmpeg')
return {'code': 200, 'Msg': 'success'}
