def db_get_list(key_list: Sequence[str],
allow_not_found=True,
fail=True,
model=None):
if len(key_list) == 0:
return []
start = time.time()
orig_ret = db_model.mget(key_list)
cost = time.time() - start
if cost > 0.01:
Log("耗时的db操作[%s][%s]" % (len(key_list), key_list[:10]))
ret = list(filter(lambda x: x is not None, orig_ret))
if len(ret) != len(key_list):
if model is not None:
tmp = model + ":"
assert len(list(filter(lambda x: not x.startswith(tmp),
key_list))) == 0
for i, k_v in enumerate(zip(key_list, orig_ret)):
if k_v[1] is not None:
continue
orig_ret[i] = mongo_get(k_v[0], model=model)
ret = list(filter(lambda x: x is not None, orig_ret))
if len(ret) != len(key_list):
if not allow_not_found:
if fail:
if isinstance(fail, bool):
raise Fail("少了一部分的数据")
else:
raise Fail(fail)
return ret
def get_capability(cate: str):
if cate == "GAME_CENTER":
return {
"type": "bundleIdCapabilities",
"attributes": {
"enabled": True,
"settings": [],
},
"relationships": {
"capability": {
"data": {
"type": "capabilities",
"id": "GAME_CENTER",
},
},
},
}
elif cate == "IN_APP_PURCHASE":
return {
"type": "bundleIdCapabilities",
"attributes": {
"enabled": True,
"settings": [],
},
"relationships": {
"capability": {
"data": {
"type": "capabilities",
"id": "IN_APP_PURCHASE",
},
},
},
}
else:
raise Fail("不支持的capability[%s]" % cate)
def _check_cert(cert: str, fail=False) -> bool:
_load_awwdrca()
_refresh_certs()
if cert in _certs:
return True
if fail:
raise Fail("找不到指定的证书")
return False
def mapping_add(cate, mapping, model_key):
key = "%s:%s" % (cate, mapping)
if db_model_ex.setnx(key, model_key) == 0:
orig = db_model_ex.get(key)
if orig == model_key:
return
else:
raise Fail("mapping出现覆盖[%s:%s] => [%s]" % (key, model_key, orig))
def wait():
__pub = db_session.pubsub()
__pub.subscribe("account:security:code")
expire = now() + 1200 * 1000
while now() < expire:
gevent.sleep(1)
for data in message_from_topic(__pub, is_json=True, limit=1):
return data
raise Fail("超时")
def __fetch_account(
udid: str, project: str, action: Callable[[IosAccountInfo, str, str],
bool]) -> IosAccountInfo:
"""
循环使用所有的账号
"""
for each in IosAccountInfo.objects.filter(
devices_num__lt=100).order_by("-devices_num"):
if action(each, udid, project):
return each
raise Fail("没有合适的账号了")
def db_set(key, value: str or int or bool, fail=True) -> bool:
"""
设置一个对象
"""
if db_model.set(key, value):
return value
else:
if fail:
raise Fail("写入[%s]错误" % key)
else:
return False
def db_add(key, value: str or int or bool, fail=True) -> bool:
"""
添加一个key
"""
if db_model.set(key, value, nx=True):
return True
else:
if fail:
raise Fail("写入[%s]错误" % key)
else:
return False
def acquire(self, timeout=10000, delta=10):
timeout = self.__timeout or timeout // 1000 or 1000
if db_session.set(self.__key, 1, ex=timeout, nx=False):
self.__lock = True
return
expire = now() + timeout
while now() < expire:
if db_session.set(self.__key, 1, ex=timeout, nx=False):
self.__lock = True
return
gevent.sleep(delta)
raise Fail("[%s]锁竞争失败" % self.__key)
def _newbee(_project: IosProjectInfo):
# 默认一天的时效
for _ in range(100):
import uuid
_uuid = uuid.uuid4()
if db_session.set("uuid:%s" % _uuid,
json_str({
"project": _project.project,
}),
ex=3600 * 24,
nx=True):
return str(_uuid)
raise Fail("生成失败")
def mongo_mget(key_list: Sequence[str],
model: Optional[str] = None,
active=True,
allow_not_found=True):
ret = []
for each in key_list:
tmp = mongo_get(each, model=model, active=active)
if tmp is None:
if not allow_not_found:
raise Fail("就是找不到指定的对象[%s]" % each)
else:
continue
ret.append(tmp)
return ret
def _wait_code(info: IosAccountInfo,
session: requests.Session,
ts,
save_now=False):
"""
等待二次提交的需要
"""
Log("等待[%s]二次验证" % info.account)
# last = info.security_code
expire = now() + 1200 * 1000
while now() < expire:
gevent.sleep(1)
for data in message_from_topic(__pub, is_json=True, limit=1):
if data.get("ts") > ts:
if data.get("code") and data.get("account") in {
info.account, "*"
}:
rsp = session.post(
"https://idmsa.apple.com/appleauth/auth/verify/phone/securitycode",
json={
"securityCode": {
"code": str(data.get("code")),
},
"phoneNumber": {
"id": 1,
},
"mode": "sms",
})
# Log("[%s] %s" % (rsp.status_code, rsp.json()))
# rsp = session.post("https://idmsa.apple.com/appleauth/auth/verify/trusteddevice/securitycode", json={
# "securityCode": {"code": data["code"]},
# })
if rsp.status_code != 200:
Log("账号[%s]验证校验码[%s]失败[%s]" %
(info.account, data.get("code"), rsp.status_code))
# if rsp.status_code != 204:
# Log("账号[%s]验证校验码[%s]失败[%s]" % (info.account, data.get("code"), rsp.status_code))
# continue
rsp = session.get(
"https://idmsa.apple.com/appleauth/auth/2sv/trust")
if rsp.status_code == 204:
session.cookies.update(
{"__expire": str(now() + 6 * 3600 * 1000)})
if save_now:
info.cookie = json_str(session.cookies)
info.save()
Log("账号[%s]登录成功" % info.account)
return session
raise Fail("登录二次验证超时")
def db_get(key, default=None, fail=True, model=None) -> str:
"""
获取一个对象
"""
ret = db_model.get(key)
if ret is None:
if model is not None:
ret = mongo_get(key, model=model)
if ret is None:
if default is None:
if fail:
if fail is True:
raise Fail("找不到指定的对象[%s]" % key)
else:
raise Fail(fail)
else:
# 这里的fail必须是True
if type(default) in {int, str, bool}:
db_set(key, default, fail=True)
else:
db_set_json(key, default, fail=True)
return default
else:
return ret
def db_set_json(key, value, fail=True) -> bool:
"""
设置一个对象
"""
if db_model.set(
key,
json.dumps(value,
ensure_ascii=False,
sort_keys=True,
cls=ExJSONEncoder)):
return True
else:
if fail:
raise Fail("写入[%s]错误" % key)
else:
return False
def add_device(_content: bytes, uuid: str, udid: str = ""):
_key = "uuid:%s" % uuid
_detail = str_json(db_session.get(_key) or "{}")
_account = _detail.get("account")
project = _detail["project"]
if not _detail:
raise Fail("无效的uuid[%s]" % uuid)
if not udid:
# todo: 验证来源
with Block("提取udid", fail=False):
udid = re.compile(br"<key>UDID</key>\n\s+<string>(\w+)</string>"
).findall(_content)[0].decode("utf8")
if not udid:
# 提取udid失败后删除uuid
db_session.delete(_key)
return {
"succ": False,
}
for _user in UserInfo.objects.filter(udid=udid):
_account = _user.account
if _user.project == _detail["project"]:
if uuid != _user.uuid:
Log("转移设备的[%s]的uuid[%s]=>[%s]" % (udid, uuid, _user.uuid))
uuid = _user.uuid
break
if not _account:
Log("为设备[%s]分配账号" % udid)
_account = __fetch_account(udid, project, __add_device)
else:
_account = IosAccountInfo.objects.filter(account=_account).first()
_user = UserInfo(uuid=uuid)
_user.udid = udid
_user.project = _detail["project"]
_user.account = _account.account
_user.save()
if test_env():
# 测试环境可能uuid要重复测
pass
else:
db_session.delete(_key)
__add_task(_user)
return HttpResponsePermanentRedirect(
"https://iosstore.sklxsj.com/detail.php?project=%s" % project)
def db_add_random_key(key_func: Callable,
value: str or int or bool,
retry: int = 10,
fail_msg="生成key尝试失败",
duration=None) -> str:
"""
多次生成key以达到生成不重复的key的效果
"""
if isinstance(value, dict):
value = json.dumps(value, ensure_ascii=False)
key = key_func()
if db_model.set(key, value, nx=True, ex=duration):
return key
while retry > 0:
retry -= 1
key = key_func()
if db_model.set(key, value, nx=True, ex=duration):
return key
raise Fail(fail_msg)
def db_pop(key, fail=True) -> str or None:
"""
读取并删除
"""
if db_model.move(key, 15):
# 转移到回收站
ret = db_trash.get(key)
else:
# 可能回收站已经有了
# 回归原始的操作
ret = db_model.get(key)
if ret is None:
if fail:
raise Fail("找不到指定的key[%s]" % key)
else:
return None
else:
db_model.delete(key)
if ret:
return ret
def _package(orig_file, provisioning_profile, certificate, output_path):
base = tempfile.mkdtemp()
ipa_file = os.path.join(base, "orig.ipa")
mp_file = os.path.join(base, "mp.ipa")
os.symlink(os.path.abspath(orig_file), ipa_file)
os.symlink(os.path.abspath(provisioning_profile), mp_file)
Log("准备ipa[%s]" % ipa_file)
# with Block("打包", fail=False):
# app = app_argument(ipa_file).unpack_to_dir(base)
# Log("准备provision")
# app.provision(mp_file)
# Log("构造entitlements")
# app.create_entitlements()
# Log("签名证书")
# app.sign(certificate)
# Log("重新封包")
# app.package(output_path)
# Log("ipa[%s]" % output_path)
try:
if not os.path.isfile(output_path) or True:
# 方案二打包
sh_file = os.path.join(base, "provisions.sh")
if os.path.exists("provisions.sh"):
os.symlink(os.path.abspath("provisions.sh"), sh_file)
else:
os.symlink(os.path.abspath("tools/provisions.sh"), sh_file)
_shell_run("%s -p %s -c '%s' %s " %
(sh_file, mp_file, certificate, ipa_file),
succ_only=True,
verbose=True,
debug=True,
pwd=base)
Assert(os.path.isfile(os.path.join(base, "stage", "out.ipa")),
"打包脚本失败了")
os.rename(os.path.join(base, "stage", "out.ipa"),
os.path.abspath(output_path))
if not os.path.isfile(output_path):
raise Fail("打包失败")
finally:
shutil.rmtree(base, ignore_errors=True)
def wrapper(self, req: HttpRequest, *args, **kwargs):
is_req = isinstance(req, HttpRequest)
if is_req:
orig_params = {}
orig_params.update(self.default_params)
if len(req.POST):
orig_params.update(to_simple_str_dict(req.POST))
if len(req.GET):
orig_params.update(to_simple_str_dict(req.GET))
if len(req.FILES):
for f, c in req.FILES.items():
orig_params[f] = c.read()
if len(req.COOKIES):
for k, v in req.COOKIES.items():
orig_params["$c_%s" % k] = v
_path = req.path.split("/")
if len(_path) > 3:
# 基于路径的参数
for i in range(4, len(_path), 2):
orig_params[_path[i - 1]] = _path[i]
content_type = req.META.get("CONTENT_TYPE", "")
if content_type.startswith('multipart'):
pass
elif len(req.body):
if "json" in content_type:
orig_params.update(json.loads(req.body.decode('utf8')))
elif "text" in content_type:
orig_params.update(json.loads(req.body.decode('utf8')))
req._orig_params = orig_params
else:
orig_params = dict(self.default_params)
orig_params.update(req)
# noinspection PyNoneFunctionAssignment
pre_ret = self.pre_wrapper(req, orig_params, *args, **kwargs)
if pre_ret is not None:
if pre_ret is True:
pass
else:
return pre_ret
params = {}
if len(self.params_hints):
if is_req:
for each in self.req_inject_func:
each(req, params)
for each in self.inject_func:
each(orig_params, params)
for k, hint in self.params_hints: # type:Tuple[str,any]
if k.startswith("_"):
continue
if k not in orig_params:
raise Fail("缺少参数[%s]" % k)
params[k] = get_data(orig_params[k], hint)
for each in self.last_inject_func:
each(orig_params, params)
try:
ret = self.func(**params)
if ret is None:
ret = {
"succ": True,
}
except Exception as e:
# _be_log = req._log if is_req else req.get("_log") # type:BeLog
# if _be_log:
# _be_log.ret = str(e)
Trace("出现异常", e)
raise e
finally:
# with Block("处理log部分", fail=False):
# _be_log = req._log if is_req else req.get("_log") # type:BeLog
# if _be_log:
# _be_log.save()
pass
return ret
def post(self,
title: str,
url: str,
data: Union[Dict, str] = None,
is_json=True,
log=True,
cache: Union[bool, int] = False,
csrf=False,
json_api=True,
method="POST",
is_binary=False,
ex_headers=None,
status=200):
if cache is True:
expire = 3600 * 1000
else:
expire = cache
if not self.is_login:
self.__login()
start = now()
rsp_str = "#NODATA#"
try:
if "teamId=" in url:
if "teamId=%s" % self.team_id not in url:
url = url.replace("teamId=", "teamId=%s" % self.team_id)
if cache:
rsp_str = _cache(url, data) or rsp_str
headers = {
'cookie':
to_form_url({"myacinfo": self.cookie["myacinfo"]}, split=';'),
}
if csrf:
headers.update({
'csrf': self.csrf,
'csrf_ts': self.csrf_ts,
})
if ex_headers:
headers.update(ex_headers)
if rsp_str == "#NODATA#":
if method.upper() == "GET":
rsp = requests.get(url,
params=data,
headers=headers,
timeout=3,
verify=False)
else:
rsp = requests.post(url,
data=data,
headers=headers,
timeout=3,
verify=False)
rsp_str = rsp.text
if rsp.headers.get("csrf"):
self.csrf = rsp.headers["csrf"]
self.csrf_ts = int(rsp.headers["csrf_ts"])
Assert(rsp.status_code == status,
"请求[%s]异常[%s]" % (title, rsp.status_code))
if json_api:
_data = str_json_i(rsp_str) or {}
if _data.get("resultCode") == 1100:
self.__logout()
raise Fail("登录[%s]过期了[%s][%s]" %
(self.account, _data.get("resultString"),
_data.get("userString")))
Assert(
_data.get("resultCode") == 0, "请求业务[%s]失败[%s][%s]" %
(title, _data.get("resultString"),
_data.get("userString")))
if log:
Log("apple请求[%s][%s]发送[%r]成功[%r]" %
(title, now() - start, data, rsp_str))
if is_binary:
rsp_str = base64(rsp.content)
if cache:
_set_cache(url, data, rsp_str, expire=expire)
if is_json:
return str_json(rsp_str)
elif is_binary:
return base64decode(rsp_str)
else:
return rsp_str
except Exception as e:
if log:
Log("apple请求[%s][%s]发送[%r]失败[%r]" %
(title, now() - start, data, rsp_str))
raise e
def _shell_run(cmd: str,
timeout=30000,
succ_only=False,
include_err=True,
err_last=False,
pwd=None,
verbose=False,
debug=False):
if debug:
verbose = True
if verbose:
if debug:
Log("+ [%s] [%s]" % (os.path.abspath(pwd or os.path.curdir), cmd))
else:
Log("+ [%s]" % cmd)
p = Popen(cmd,
bufsize=1,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT
if include_err and not err_last else subprocess.PIPE,
shell=True,
cwd=pwd)
expire = now() + timeout
buffer = []
buffer_err = []
while p.poll() is None:
if now() < expire:
time.sleep(0.1)
else:
p.kill()
break
if p.stdout.readable():
buffer.extend(p.stdout.readlines())
if include_err:
if err_last:
if p.stderr.readable():
buffer_err.extend(p.stderr.readlines())
if succ_only:
if p.returncode == 0:
pass
else:
raise Fail("命令[%s]执行失败" % cmd)
if p.stdout and p.stdout.readable():
buffer.extend(p.stdout.readlines())
if p.stderr and p.stderr.readable():
buffer_err.extend(p.stderr.readlines())
# todo: 采用yield后会导致不执行...
# if debug:
# for each in map(lambda x: x.decode("utf8"), buffer): # type:str
# Log("- %s" % each.rstrip("\r\n"))
# yield each.rstrip("\r\n")
# if err_last:
# for each in map(lambda x: x.decode("utf8"), buffer_err): # type:str
# Log("* %s" % each.rstrip("\r\n"))
# yield each.rstrip("\r\n")
# else:
# for each in map(lambda x: x.decode("utf8"), buffer): # type:str
# yield each.rstrip("\r\n")
# if err_last:
# for each in map(lambda x: x.decode("utf8"), buffer_err): # type:str
# yield each.rstrip("\r\n")
if debug:
tmp = []
for each in map(lambda x: x.decode("utf8"), buffer): # type:str
Log("- %s" % each.rstrip("\r\n"))
tmp.append(each.rstrip("\r\n"))
if err_last:
for each in map(lambda x: x.decode("utf8"),
buffer_err): # type:str
Log("* %s" % each.rstrip("\r\n"))
tmp.append(each.rstrip("\r\n"))
return tmp
else:
tmp = []
for each in map(lambda x: x.decode("utf8"), buffer): # type:str
tmp.append(each.rstrip("\r\n"))
if err_last:
for each in map(lambda x: x.decode("utf8"),
buffer_err): # type:str
tmp.append(each.rstrip("\r\n"))
return tmp
def __add_device(account: IosAccountInfo, udid: str, project: str) -> bool:
title = "设备%s" % udid
_config = IosAccountHelper(account)
try:
if udid not in account.devices:
# 先注册设备
ret = _config.post(
"验证设备udid",
"https://developer.apple.com/services-account/QH65B2/account/ios/device/validateDevices.action?teamId=",
{
"deviceNames": title,
"deviceNumbers": udid,
"register": "single",
"teamId": _config.team_id,
},
cache=True)
if len(ret["failedDevices"]) == 0:
added = False
else:
if "already exists on this team" in ret["failedDevices"][0]:
# 已经添加过了
Log("[%s]已经添加过的设备了[%s]" % (account.account, udid))
added = True
__list_all_devices(_config)
else:
raise Fail("验证udid请求失败[%s][%s]" %
(udid, ret["validationMessages"]))
if not added:
__list_all_devices(_config)
ret = _config.post(
"添加设备",
"https://developer.apple.com/services-account/QH65B2/account/ios/device/addDevices.action?teamId=%s"
% _config.team_id, {
"deviceClasses": "iphone",
"deviceNames": title,
"deviceNumbers": udid,
"register": "single",
"teamId": _config.team_id,
},
csrf=True)
Assert(ret["resultCode"] == 0, "添加udid请求失败[%s]" % udid)
Assert(not ret["validationMessages"], "添加udid请求失败[%s]" % udid)
Assert(ret["devices"], "添加udid请求失败[%s]" % udid)
device = ret["devices"][0] # type: Dict
_reg_device(_config.account, device["deviceId"],
device["deviceNumber"], device["model"],
device["serialNumber"])
with Block("更新"):
ret, _info = __list_all_profile(_config, project)
if not _info:
_info = IosProfileInfo()
_info.sid = "%s:%s" % (_config.account, project)
_info.app = _config.account
_info.devices = "[]"
_info.devices_num = 0
_info.project = project
devices = str_json(_info.devices) # type: List[str]
if udid in devices:
pass
else:
devices.append(udid)
with Block("默认全开当期的设备"):
# noinspection PyTypeChecker
devices = list(
set(devices +
list(str_json(_config.info.devices).keys())))
_cert = _get_cert(_config.info)
_app = _get_app(_config, project)
found = False
for each in ret["provisioningProfiles"]: # type: Dict
if each["name"] != "专用 %s" % project:
continue
# todo: 过期更新
ret = _config.post(
"更新ProvisioningProfile",
"https://developer.apple.com/services-account/QH65B2/account/ios/profile/regenProvisioningProfile.action?teamId=",
data={
"provisioningProfileId":
each["provisioningProfileId"],
"distributionType":
"limited",
"subPlatform":
"",
"returnFullObjects":
False,
"provisioningProfileName":
each["name"],
"appIdId":
_app.app_id_id,
"certificateIds":
_cert.cert_req_id,
"deviceIds":
",".join(
map(
lambda x: str_json(_config.info.devices)[
x], devices)),
},
csrf=True)
Assert(ret["resultCode"] == 0)
_info.devices = json_str(devices)
_info.profile_id = each["provisioningProfileId"]
# noinspection PyTypeChecker
_info.profile = ret["provisioningProfile"][
"encodedProfile"]
_info.expire = _to_dt(
ret["provisioningProfile"]["dateExpire"])
_info.save()
found = True
Log("更新证书[%s]添加设备[%s][%s]成功" %
(project, udid, len(devices)))
break
if not found:
ret = _config.post(
"创建ProvisioningProfile",
"https://developer.apple.com/services-account/QH65B2/account/ios/profile/createProvisioningProfile.action?teamId=",
data={
"subPlatform":
"",
"certificateIds":
_cert.cert_req_id,
"deviceIds":
",".join(
map(
lambda x: str_json(_config.info.devices)[
x], devices)),
"template":
"",
"returnFullObjects":
False,
"distributionTypeLabel":
"distributionTypeLabel",
"distributionType":
"limited",
"appIdId":
_app.app_id_id,
"appIdName":
_app.name,
"appIdPrefix":
_app.prefix,
"appIdIdentifier":
_app.identifier,
"provisioningProfileName":
"专用 %s" % project,
},
csrf=True)
Assert(ret["resultCode"] == 0)
# noinspection PyTypeChecker
_info.profile_id = ret["provisioningProfile"][
"provisioningProfileId"]
# noinspection PyTypeChecker
_info.profile = ret["provisioningProfile"][
"encodedProfile"]
_info.expire = _to_dt(
ret["provisioningProfile"]["dateExpire"])
_info.save()
Log("添加证书[%s]添加设备[%s][%s]成功" %
(project, udid, len(devices)))
except Exception as e:
Trace("添加设备出错了[%s]" % e, e)
return False
return True
def _set_cache(url: str, data: Dict, content: str, expire: int):
db_session.set("http:cache:%s:%s" % (url, to_form_url(data)),
content,
ex=expire // 1000)
__pub = db_session.pubsub()
# noinspection PyBroadException
try:
db_session.info()
__pub.subscribe("account:security:code")
except:
if os.environ.get("REDIS_HOST"):
Fail("redis[%s:%s]没准备好" %
(os.environ.get("REDIS_HOST"), os.environ.get("REDIS_PORT")))
else:
pass
def publish_security_code(account: str, code: str, ts: int):
db_session.publish(
"account:security:code",
json_str({
"account": account,
"code": code,
"ts": ts,
}))
def get_capability(cate: str):
