def format_information(self, result):
if result:
information = json.loads(result)
Operation("[+] %s - %s - %s:" %
(self.region, information["Namespace"], self.name),
fg="cyan").echo()
Operation("Name: %s" % self.name).out_infor()
Operation("Version: %s" %
information["FunctionVersion"]).out_infor()
Operation("Status: %s" % information["Status"]).out_infor()
Operation("FunctionId: %s" % information["FunctionId"]).out_infor()
Operation("Region: %s" % self.region).out_infor()
Operation("Namespace: %s" % information["Namespace"]).out_infor()
Operation("Runtime: %s" % information["Runtime"]).out_infor()
release_serviceid_list = []
if information["Triggers"]:
Operation(" Trigger Information: ", fg="cyan").echo()
for eve_trigger in information["Triggers"]:
trigger_type = eve_trigger['Type']
# release apigateway service_id list
try:
if trigger_type == 'apigw':
release_serviceid_list.append(
json.loads(eve_trigger['TriggerDesc'])
["service"]["serviceId"])
except Exception as e:
pass
msg = u" > %s - %s:" % (
text(trigger_type).upper(),
text(eve_trigger["TriggerName"]))
Operation(msg, fg="cyan").echo()
self.recursion_dict(eve_trigger, 6)
def _do_deploy_trigger(self, func, func_name, func_ns, region=None):
proper = func.get(tsmacro.Properties, {})
events = proper.get(tsmacro.Events, {})
hasError = None
for trigger in events:
err = ScfClient(region).deploy_trigger(events[trigger], trigger,
func_name, func_ns)
if err is not None:
hasError = err
if sys.version_info[0] == 3:
s = err.get_message()
else:
s = err.get_message().encode("UTF-8")
if "Param error The path+method already exists under the service" not in str(
s):
if err.get_request_id():
Operation(
"Deploy trigger '{name}' failure. Error: {e}. RequestId: {id}"
.format(name=trigger, e=s,
id=err.get_request_id())).warning()
else:
Operation(
"Deploy trigger '{name}' failure. Error: {e}.".
format(
name=trigger,
e=s,
)).warning()
continue
Operation("Deploy trigger '{name}' success".format(
name=trigger)).success()
def get_information(self):
if self.name:
try:
function_info = self.information_client(
self.namespace, self.name)
if function_info:
self.format_information(function_info)
else:
Operation(
"Data not available, please check Region/Namespace/FunctionName."
).exception()
except:
Operation("Could not find function resource: %s - %s - %s" %
(self.region, self.namespace, self.name)).warning()
Operation(
"You can view the list of functions by: scf function list, and select the correct function resource for information query"
).warning()
raise Information("No Function Resource Exception")
else:
Operation(
"Function Name to be queried must be specified").warning()
Operation(
"You can view the list of functions by: scf function list, and select the correct function resource for information query"
).warning()
raise InformationException("No Function Resource Exception")
def show(region, namespace):
if region and region not in REGIONS:
raise ArgsException(
"region {r} not exists ,please select from{R}".format(
r=region, R=REGIONS))
# return
rep = ScfClient(region).get_ns(namespace)
if not rep:
return False
functions = ScfClient(region).list_function(namespace)
if not functions:
# click.secho("Region:%s \nNamespace:%s " % (region, namespace), fg="green")
# click.secho("no function exists\n", fg="red")
return
Operation("Region:%s" % (region)).process()
Operation("Namespace:%s " % (namespace)).process()
click.secho(
"%-20s %-15s %-20s %-20s %-60s" %
("Runtime", "Status", "AddTime", "ModTime", "FunctionName"))
for function in functions:
click.secho(
"%-20s %-24s %-20s %-20s %-60s" %
(function.Runtime, List.status(function.Status),
function.AddTime, function.ModTime, function.FunctionName))
click.secho("\n")
def show(region, namespace, name):
if region and region not in REGIONS:
raise ArgsException("region {r} not exists ,please select from{R}".format(r=region, R=REGIONS))
if not region:
region = UserConfig().region
rep = ScfClient(region).get_ns(namespace)
if not rep:
raise NamespaceException("Region {r} not exist namespace {n}".format(r=region, n=namespace))
functions = ScfClient(region).get_function(function_name=name, namespace=namespace)
if not functions:
raise FunctionNotFound("Region {r} namespace {n} not exist function {f}".format(r=region, n=namespace, f=name))
return
Operation("Region:%s" % (region)).process()
Operation("Namespace:%s " % (namespace)).process()
Operation("Function:%s " % (name)).process()
testmodels = ScfClient(region).list_func_testmodel(functionName=name, namespace=namespace)
if not testmodels:
raise NamespaceException("This function not exist event".format(f=name))
click.secho("%-20s %-20s %-20s" % ("TestmodelsName", "AddTime", "ModTime"))
for testmodel in testmodels:
res = ScfClient(region).get_func_testmodel(functionName=name, namespace=namespace, testModelName=testmodel)
click.secho("%-20s %-20s %-20s" % (testmodel, res['CreatedTime'], res['ModifiedTime']))
click.secho("\n")
def status(status_name):
if status_name == "Active":
return Operation(status_name, fg="green").style()
elif "Failed" in status_name:
return Operation(status_name, fg="red").style()
else:
return Operation(status_name, fg="blue").style()
def invoke(name, region, namespace, eventdata, type, no_color):
"""
\b
Invoke the SCF remote function.
\b
Common usage:
\b
* Invoke the function test in ap-guangzhou and in namespace default
$ scf remote invoke --name test --region ap-guangzhou --namespace default
"""
type_dict = {"sync": "RequestResponse", "async": "Event"}
logtype = "tail"
if type.lower() not in type_dict:
Operation("Log type must in {l}".format(l=INVOCATION_TYPE)).warning()
return
if type.lower == "async":
Operation(
'invoke start ,you can get the invoke logs by excute `scf logs -r %s -ns %s -n %s`'
% (region, namespace, name)).information()
if eventdata:
try:
eventdata = get_data(eventdata)
except Exception as e:
raise EventFileException("Read file error: %s" % (str(e)))
else:
eventdata = json.dumps({"key1": "value1", "key2": "value2"})
Invoke.do_cli(name, region, namespace, eventdata, logtype,
type_dict[type.lower()])
def do_deploy_testmodel(functionName, namespace, region, forced, event,
event_name):
# 获取失败抛出异常会直接创建新模版
# 获取成功代表已有模版，根据force是否覆盖
try:
ScfClient(region).get_func_testmodel(functionName=functionName,
testModelName=event_name,
namespace=namespace)
if not forced:
Operation("Eventdata {%s} exist in remote" %
event_name).exception()
return True
else:
Operation("Eventdata {%s} exist in remote,updating event..." %
event_name).process()
ScfClient(region).update_func_testmodel(
functionName=functionName,
testModelValue=event,
testModelName=event_name,
namespace=namespace)
Operation("Eventdata {%s} update success!" %
event_name).success()
except:
Operation("Updateing eventdata {%s}..." % event_name).process()
ScfClient(region).create_func_testmodel(functionName=functionName,
testModelValue=event,
testModelName=event_name,
namespace=namespace)
Operation("Eventdata {%s} update success!" % event_name).success()
def recursion_dict(self, information, num):
for eveKey, eveValue in information.items():
try:
eveValue = json.loads(eveValue)
except:
pass
finally:
if isinstance(eveValue, dict):
Operation(" " * num + "%s:" % (str(eveKey))).out_infor()
self.recursion_dict(eveValue, num + 2)
else:
Operation(" " * num + "%s: %s" % (str(eveKey), str(eveValue))).out_infor()
def pull_image(self, image):
try:
if not self._is_quiet:
Operation('pull image %s...' % image, nl=False).Operation
for _ in self._docker_client.api.pull(image, stream=True, decode=True):
if not self._is_quiet:
Operation('.', nl=False).echo()
if not self._is_quiet:
Operation('\n').echo()
except docker.errors.APIError as e:
Operation(e, err_msg=traceback.format_exc(), level="ERROR").no_output()
raise Exception('pull the docker image %s failed, %s' % (image, str(e)))
def format_information(self):
result = self.get_information()
if result:
information = json.loads(result)
click.secho(u"[+] Function Base Information: ", fg="cyan")
Operation("Name: %s" % self.function).out_infor()
Operation("Version: %s" %
information["FunctionVersion"]).out_infor()
Operation("Status: %s" % information["Status"]).out_infor()
Operation("FunctionId: %s" % information["FunctionId"]).out_infor()
Operation("Region: %s" % self.region).out_infor()
Operation("Namespace: %s" % information["Namespace"]).out_infor()
Operation("Runtime: %s" % information["Runtime"]).out_infor()
release_serviceid_list = []
if information["Triggers"]:
click.secho(u"[+] Trigger Information: ", fg="cyan")
for eve_trigger in information["Triggers"]:
trigger_type = eve_trigger['Type']
# release apigateway service_id list
try:
if trigger_type == 'apigw':
release_serviceid_list.append(
json.loads(eve_trigger['TriggerDesc'])
["service"]["serviceId"])
except Exception as e:
pass
msg = u" > %s - %s:" % (text(trigger_type).upper(),
text(
eve_trigger["TriggerName"]))
click.secho(click.style(msg), fg="cyan")
self.recursion_dict(eve_trigger, 2)
# yaml apigateway service_id list
function = self.resources[self.namespace][self.function]
proper = function.get(tsmacro.Properties, {})
events = proper.get(tsmacro.Events, {})
yaml_serviceid_list = []
for eve_event in events:
if "ServiceId" in events[eve_event]['Properties']:
yaml_serviceid_list.append(
events[eve_event]['Properties']["ServiceId"])
remind_serviceid_list = []
for eve_service_id in release_serviceid_list:
if eve_service_id not in yaml_serviceid_list:
remind_serviceid_list.append(eve_service_id)
if remind_serviceid_list:
Operation(
"If you don't want to create the new gateway next time."
).information()
Operation("Please add these ServiceId into the YAML: " +
", ".join(remind_serviceid_list)).information()
def pull_image(self, image):
try:
if not self._is_quiet:
Operation('pull image %s...' % image, nl=False).Operation
for _ in self._docker_client.api.pull(image,
stream=True,
decode=True):
if not self._is_quiet:
Operation('.', nl=False).echo()
if not self._is_quiet:
Operation('\n').echo()
except docker.errors.APIError as e:
raise Exception('pull the docker image %s failed, %s' %
(image, str(e)))
def output_msg(local_version, release_version, message):
upgrade_command_msg_start = Operation(""" | """, fg="green").style()
upgrade_command_msg_middl = Operation("""pip install -U scf""", fg="yellow").style()
upgrade_command_msg_end = Operation(""" |""", fg="green").style()
Operation(""" ----------------------------------------------------""", fg="green").echo()
Operation(""" | Upgrade reminder |""", fg="green").echo()
Operation(""" | Latest version：%7s , Your version: %7s |""" % (release_version, local_version),
fg="green").echo()
Operation(""" | If you want to upgrade, you can use the command: |""", fg="green").echo()
Operation(upgrade_command_msg_start + upgrade_command_msg_middl + upgrade_command_msg_end).echo()
Operation(""" ----------------------------------------------------""", fg="green").echo()
if message:
for eve_msg in message:
Operation(eve_msg).information()
def fetch_log(self, startime, endtime, count, tail=False):
for logs in self.__fetch_log(startime, endtime, count, tail):
if len(logs) == 0 and not tail:
msg = "There is no data during this time period. You can try to adjust the start-time, end-time, duration, etc. to view a larger range of logs. For example:\n $ scf logs -s xxxx-xx-xx 00:00:00 -e xxxx-xx-xx 00:00:10\n $ scf logs -n function -d 10"
Operation(msg).information()
for log in logs:
Operation("Log startTime: %s" % str(log.StartTime)).process()
if log.RetCode == 0:
click.secho(u"%s" % (text(log.Log)).replace("\n\n", "\n"))
else:
click.secho(u"%s" % (text(log.Log)).replace("\n\n", "\n"), fg="red")
click.secho("\n")
def file_size_infor(self, size):
# click.secho(str(size))
if size >= 20 * 1024 * 1024:
Operation(
'Your package is too large and needs to be uploaded via COS.'
).warning()
Operation(
'You can use --cos-bucket BucketName to specify the bucket, or you can use the "scf configure set" to set the default to open the cos upload.'
).warning()
raise UploadFailed("Upload faild")
elif size >= 8 * 1024 * 1024:
Operation(
"Package size is over 8M, it is highly recommended that you upload using COS. "
).information()
return
def list_ns(self):
try:
resp = self._client_ext.ListNamespaces()
namespaces = resp.get("Namespaces", [])
return namespaces
except TencentCloudSDKException as err:
Operation(err, err_msg=traceback.format_exc(),
level="ERROR").no_output()
if sys.version_info[0] == 3:
s = err.get_message()
else:
s = err.get_message().encode("UTF-8")
Operation(
"list namespace failure. Error: {e}.".format(e=s)).warning()
return None
def update_code(self, func, func_name, func_ns):
try:
self.update_func_code(func, func_name, func_ns)
Operation("Update funcion code success.").success()
return True
except TencentCloudSDKException as err:
return err
def get_data(self, func_name, start_time, end_time, metric):
try:
req = models.GetMonitorDataRequest()
req.Period = self.period
req.MetricName = metric
req.Namespace = MonitorClient.ProductNS
req.StartTime = start_time
req.EndTime = end_time
req.Instances = []
version = models.Dimension()
version.Name = "version"
version.Value = "$latest"
funcname = models.Dimension()
funcname.Name = 'functionName'
funcname.Value = func_name
params = models.Instance()
params.Dimensions = []
params.Dimensions.append(version)
params.Dimensions.append(funcname)
req.Instances.append(params)
return self._client.GetMonitorData(req)
except TencentCloudSDKException as err:
if sys.version_info[0] == 3:
s = err.get_message()
else:
s = err.get_message().encode("UTF-8")
Operation("get data '{name}' failure. Error: {code} {e}.".format(
name=func_name, code=err.get_code(), e=s)).warning()
return None
def do_cli(region, namespace, name, dir, forced):
eventdatalist = []
if not os.path.exists(dir):
raise EventFileNotFoundException("Event Dir Not Exists")
elif os.path.isfile(dir):
if not dir.endswith('.json'):
raise EventFileNameFormatException(
'Please check your json file name endwith `.json`.')
eventdata = Update.get_event_from_file(dir)
if eventdata:
eventdatalist.append(eventdata)
elif os.path.isdir(dir):
for path in os.listdir(dir):
filepath = os.path.join(dir, path)
if os.path.isfile(filepath) and filepath.endswith('.json'):
eventdata = Update.get_event_from_file(filepath)
if eventdata:
eventdatalist.append(eventdata)
if len(eventdatalist):
Update.update_event_data(region=region,
namespace=namespace,
name=name,
eventdatalist=eventdatalist,
forced=forced)
else:
Operation("Eventdata file not found").exception()
def deploy_func(self, func, func_name, func_ns, forced):
try:
# SERVICE_RUNTIME_SUPPORT_LIST = ["Nodejs8.9-service"]
# if 'Type' in func['Properties'] and func['Properties']['Type'] == 'HTTP' and \
# func['Properties']['Runtime'] in SERVICE_RUNTIME_SUPPORT_LIST:
# self.create_service(func, func_name, func_ns)
# else:
self.create_func(func, func_name, func_ns)
return
except TencentCloudSDKException as err:
if err.code in [
"ResourceInUse.Function", "ResourceInUse.FunctionName"
] and forced:
pass
else:
return err
Operation("{ns} {name} already exists, update it now".format(
ns=func_ns, name=func_name)).process()
try:
# if 'Type' in func['Properties'] and func['Properties']['Type'] == 'HTTP' and \
# func['Properties']['Runtime'] in SERVICE_RUNTIME_SUPPORT_LIST:
# self.update_service_config(func, func_name, func_ns)
# self.update_service_code(func, func_name, func_ns)
# else:
self.update_func_config(func, func_name, func_ns)
self.update_func_code(func, func_name, func_ns)
except TencentCloudSDKException as err:
return err
return
def upload_file2cos2(self, bucket, file, key, max_thread=5):
try:
response = self._client.upload_file(
Bucket=bucket,
LocalFilePath=file,
Key=key,
MAXThread=max_thread,
# EnableMD5=md5,
Metadata={
'x-cos-acl': 'public-read',
'Content-Type': 'application/x-zip-compressed',
})
if not response['ETag']:
return "Upload func package failed [No ETag]."
except Exception as e:
Operation(e, err_msg=traceback.format_exc(),
level="ERROR").no_output()
try:
if "<?xml" in str(e):
error_code = re.findall("<Code>(.*?)</Code>", str(e))[0]
error_message = re.findall("<Message>(.*?)</Message>",
str(e))[0]
return "COS client error code: %s, message: %s" % (
error_code, error_message)
else:
return str(e)
finally:
# raise UploadToCosFailed("Upload func package failed.")
return str(e)
return True
def has_image(self, image):
try:
self._docker_client.images.get(image)
return True
except docker.errors.ImageNotFound as e:
Operation(e, err_msg=traceback.format_exc(), level="ERROR").no_output()
return False
def get_bucket(self, bucket):
'''
通过get_bucket_list()方法，获得bucket list，再根据地域和bucket筛选目标bucket。
这里可以考虑使用head_bucket(Bucket)，但是目前这个方法异常：qcloud_cos.cos_exception.CosServiceError / 2019-07-28
后期这个方法恢复，可以作为一个优化点，对该部分进行优化。
:param bucket: str bucket名称
:return: 1表示找到了指定Region的Bucket，0表示没找到，error表示错误
'''
try:
temp_data = self.get_bucket_list()
# print(temp_data)
if temp_data[0] == 0:
bucket_list = temp_data[1]
for eve_bucket in bucket_list:
if eve_bucket["Location"] == self._region and eve_bucket[
"Name"] == bucket:
return 1
return 0
else:
error = temp_data[1]
return error
except Exception as e:
Operation(e, err_msg=traceback.format_exc(),
level="ERROR").no_output()
return e
def get(**kwargs):
'''
\b
Get your account parameters.
\b
Common usage:
\b
* Get the configured information
$ scf configure get
'''
uc = UserConfig()
def set_true(k):
kwargs[k] = True
bools = [v for k, v in kwargs.items()]
if not reduce(lambda x, y: bool(x or y), bools):
list(map(set_true, kwargs))
attrs = uc.get_attrs(kwargs)
msg = "{} config:".format(UserConfig.API)
for attr in sorted(attrs):
attr_value = attrs[attr]
if attr == "secret-id":
attr_value = "*" * 32 + attr_value[32:]
elif attr == "secret-key":
attr_value = "*" * 28 + attr_value[28:]
msg += click.style("\n[-] ", fg="cyan") + click.style(
"{} = {}".format(attr, attr_value), fg="cyan")
Operation(msg.strip()).process()
def fetch_log_tail_c(self, startime, endtime, count, tail):
log_stack = []
for logs in self.__fetch_log(startime,
endtime,
count,
tail,
order="desc"):
for log in logs:
log_stack.append(log)
for i in range(len(log_stack) - 1, -1, -1):
log = log_stack[i]
Operation(log.StartTime, fg="green").echo()
if log.RetCode == 0:
Operation(log.Log).echo()
else:
Operation(log.Log, fg="red").echo()
def deploy_func(self, func, func_name, func_ns, forced):
'''
:param func:
:param func_name:
:param func_ns:
:param forced:
:return: 0 : success
1 : error in-use
2 : only in-use
e : error
'''
try:
# SERVICE_RUNTIME_SUPPORT_LIST = ["Nodejs8.9-service"]
# if 'Type' in func['Properties'] and func['Properties']['Type'] == 'HTTP' and \
# func['Properties']['Runtime'] in SERVICE_RUNTIME_SUPPORT_LIST:
# self.create_service(func, func_name, func_ns)
# else:
self.create_func(func, func_name, func_ns)
return 0
except TencentCloudSDKException as err:
Operation(err, err_msg=traceback.format_exc(),
level="ERROR").no_output()
if err.code in [
"ResourceInUse.Function", "ResourceInUse.FunctionName"
]:
return 1 if forced else 2
else:
return err
def upload_file2cos2(self, bucket, file, key, md5):
# save funcs in the func directory
try:
response = self._client.upload_file(Bucket=bucket,
LocalFilePath=file,
Key=key,
EnableMD5=md5,
Metadata={
'x-cos-acl': 'public-read',
'Content-Type': 'application/x-zip-compressed',
})
if not response['ETag']:
raise UploadToCosFailed("Upload func package failed")
except Exception as e:
# error_msg = ""
# if "<?xml" in e.message:
# msg_dict = xmltodict.parse(e.message)
# if isinstance(msg_dict, dict):
# error_msg_dict = msg_dict.get("Error", {})
# error_msg = error_msg_dict.get("Code", "") + ", " + error_msg_dict.get("Message", "")
# else:
# error_msg = e.message
# raise UploadToCosFailed("Upload func package failed. {} ".format(error_msg))
try:
if "<?xml" in str(e):
error_code = re.findall("<Code>(.*?)</Code>", str(e))[0]
error_message = re.findall("<Message>(.*?)</Message>", str(e))[0]
Operation("COS client error code: %s, message: %s" % (error_code, error_message)).warning()
finally:
raise UploadToCosFailed("Upload func package failed.")
code_uri_in_cos = bucket + '/' + key
return code_uri_in_cos
def remove_trigger(self, trigger, func_name, func_ns):
try:
self.delete_trigger(trigger, func_name, func_ns)
except TencentCloudSDKException as err:
Operation(err, err_msg=traceback.format_exc(),
level="ERROR").no_output()
return err
def list_func_testmodel(self, functionName, namespace):
try:
resp = self._client_ext.ListFunctionTestModels(
functionName=functionName, namespace=namespace)
testmodels = resp.get("TestModels", [])
return testmodels
except TencentCloudSDKException as err:
Operation(err, err_msg=traceback.format_exc(),
level="ERROR").no_output()
if sys.version_info[0] == 3:
s = err.get_message()
else:
s = err.get_message().encode("UTF-8")
Operation(
"list testmodels failure. Error: {e}.".format(e=s)).warning()
return None
def invoke(self, event=None, stdout=None, stderr=None):
image = self.get_image()
cmd = [self.get_handler()]
code_abs_path = self.get_code_abs_path()
memory = self.get_memory()
entry = self.get_entry_point()
ports ={self._debug_options.debug_port: self._debug_options.debug_port} \
if self._debug_options else None
envs = self.get_envs(event)
timer = None
with self._get_code(code_abs_path) as code_dir:
self._container = Container(image=image,
cmd=cmd,
work_dir=self._WORK_DIR,
host_dir=code_dir,
mem=memory,
env_vars=envs,
entrypoint=entry,
ports=ports)
try:
self._container_manager.run(self._container)
timer = self._wait_timeout(self._container, self.get_timeout(),
bool(self._debug_options))
self._container.get_logs(stdout=stdout, stderr=stderr)
except KeyboardInterrupt:
Operation('Abort function execution',
err_msg=traceback.format_exc(),
level="ERROR").echo()
except Exception as err:
Operation('Invoke Failed.',
fg="red",
err_msg=traceback.format_exc(),
level="ERROR").echo()
raise InvokeException('Invoke error:%s' % str(err))
finally:
if timer:
timer.cancel()
self._container.delete()
if self._thread_err_msg != "":
raise TimeoutException(self._thread_err_msg)
