def db_get_list(key_list: Sequence[str], allow_not_found=True, fail=True, model=None): if len(key_list) == 0: return [] start = time.time() orig_ret = db_model.mget(key_list) cost = time.time() - start if cost > 0.01: Log("耗时的db操作[%s][%s]" % (len(key_list), key_list[:10])) ret = list(filter(lambda x: x is not None, orig_ret)) if len(ret) != len(key_list): if model is not None: tmp = model + ":" assert len(list(filter(lambda x: not x.startswith(tmp), key_list))) == 0 for i, k_v in enumerate(zip(key_list, orig_ret)): if k_v[1] is not None: continue orig_ret[i] = mongo_get(k_v[0], model=model) ret = list(filter(lambda x: x is not None, orig_ret)) if len(ret) != len(key_list): if not allow_not_found: if fail: if isinstance(fail, bool): raise Fail("少了一部分的数据") else: raise Fail(fail) return ret
def get_capability(cate: str): if cate == "GAME_CENTER": return { "type": "bundleIdCapabilities", "attributes": { "enabled": True, "settings": [], }, "relationships": { "capability": { "data": { "type": "capabilities", "id": "GAME_CENTER", }, }, }, } elif cate == "IN_APP_PURCHASE": return { "type": "bundleIdCapabilities", "attributes": { "enabled": True, "settings": [], }, "relationships": { "capability": { "data": { "type": "capabilities", "id": "IN_APP_PURCHASE", }, }, }, } else: raise Fail("不支持的capability[%s]" % cate)
def _check_cert(cert: str, fail=False) -> bool: _load_awwdrca() _refresh_certs() if cert in _certs: return True if fail: raise Fail("找不到指定的证书") return False
def mapping_add(cate, mapping, model_key): key = "%s:%s" % (cate, mapping) if db_model_ex.setnx(key, model_key) == 0: orig = db_model_ex.get(key) if orig == model_key: return else: raise Fail("mapping出现覆盖[%s:%s] => [%s]" % (key, model_key, orig))
def wait(): __pub = db_session.pubsub() __pub.subscribe("account:security:code") expire = now() + 1200 * 1000 while now() < expire: gevent.sleep(1) for data in message_from_topic(__pub, is_json=True, limit=1): return data raise Fail("超时")
def __fetch_account( udid: str, project: str, action: Callable[[IosAccountInfo, str, str], bool]) -> IosAccountInfo: """ 循环使用所有的账号 """ for each in IosAccountInfo.objects.filter( devices_num__lt=100).order_by("-devices_num"): if action(each, udid, project): return each raise Fail("没有合适的账号了")
def db_set(key, value: str or int or bool, fail=True) -> bool: """ 设置一个对象 """ if db_model.set(key, value): return value else: if fail: raise Fail("写入[%s]错误" % key) else: return False
def db_add(key, value: str or int or bool, fail=True) -> bool: """ 添加一个key """ if db_model.set(key, value, nx=True): return True else: if fail: raise Fail("写入[%s]错误" % key) else: return False
def acquire(self, timeout=10000, delta=10): timeout = self.__timeout or timeout // 1000 or 1000 if db_session.set(self.__key, 1, ex=timeout, nx=False): self.__lock = True return expire = now() + timeout while now() < expire: if db_session.set(self.__key, 1, ex=timeout, nx=False): self.__lock = True return gevent.sleep(delta) raise Fail("[%s]锁竞争失败" % self.__key)
def _newbee(_project: IosProjectInfo): # 默认一天的时效 for _ in range(100): import uuid _uuid = uuid.uuid4() if db_session.set("uuid:%s" % _uuid, json_str({ "project": _project.project, }), ex=3600 * 24, nx=True): return str(_uuid) raise Fail("生成失败")
def mongo_mget(key_list: Sequence[str], model: Optional[str] = None, active=True, allow_not_found=True): ret = [] for each in key_list: tmp = mongo_get(each, model=model, active=active) if tmp is None: if not allow_not_found: raise Fail("就是找不到指定的对象[%s]" % each) else: continue ret.append(tmp) return ret
def _wait_code(info: IosAccountInfo, session: requests.Session, ts, save_now=False): """ 等待二次提交的需要 """ Log("等待[%s]二次验证" % info.account) # last = info.security_code expire = now() + 1200 * 1000 while now() < expire: gevent.sleep(1) for data in message_from_topic(__pub, is_json=True, limit=1): if data.get("ts") > ts: if data.get("code") and data.get("account") in { info.account, "*" }: rsp = session.post( "https://idmsa.apple.com/appleauth/auth/verify/phone/securitycode", json={ "securityCode": { "code": str(data.get("code")), }, "phoneNumber": { "id": 1, }, "mode": "sms", }) # Log("[%s] %s" % (rsp.status_code, rsp.json())) # rsp = session.post("https://idmsa.apple.com/appleauth/auth/verify/trusteddevice/securitycode", json={ # "securityCode": {"code": data["code"]}, # }) if rsp.status_code != 200: Log("账号[%s]验证校验码[%s]失败[%s]" % (info.account, data.get("code"), rsp.status_code)) # if rsp.status_code != 204: # Log("账号[%s]验证校验码[%s]失败[%s]" % (info.account, data.get("code"), rsp.status_code)) # continue rsp = session.get( "https://idmsa.apple.com/appleauth/auth/2sv/trust") if rsp.status_code == 204: session.cookies.update( {"__expire": str(now() + 6 * 3600 * 1000)}) if save_now: info.cookie = json_str(session.cookies) info.save() Log("账号[%s]登录成功" % info.account) return session raise Fail("登录二次验证超时")
def db_get(key, default=None, fail=True, model=None) -> str: """ 获取一个对象 """ ret = db_model.get(key) if ret is None: if model is not None: ret = mongo_get(key, model=model) if ret is None: if default is None: if fail: if fail is True: raise Fail("找不到指定的对象[%s]" % key) else: raise Fail(fail) else: # 这里的fail必须是True if type(default) in {int, str, bool}: db_set(key, default, fail=True) else: db_set_json(key, default, fail=True) return default else: return ret
def db_set_json(key, value, fail=True) -> bool: """ 设置一个对象 """ if db_model.set( key, json.dumps(value, ensure_ascii=False, sort_keys=True, cls=ExJSONEncoder)): return True else: if fail: raise Fail("写入[%s]错误" % key) else: return False
def add_device(_content: bytes, uuid: str, udid: str = ""): _key = "uuid:%s" % uuid _detail = str_json(db_session.get(_key) or "{}") _account = _detail.get("account") project = _detail["project"] if not _detail: raise Fail("无效的uuid[%s]" % uuid) if not udid: # todo: 验证来源 with Block("提取udid", fail=False): udid = re.compile(br"<key>UDID</key>\n\s+<string>(\w+)</string>" ).findall(_content)[0].decode("utf8") if not udid: # 提取udid失败后删除uuid db_session.delete(_key) return { "succ": False, } for _user in UserInfo.objects.filter(udid=udid): _account = _user.account if _user.project == _detail["project"]: if uuid != _user.uuid: Log("转移设备的[%s]的uuid[%s]=>[%s]" % (udid, uuid, _user.uuid)) uuid = _user.uuid break if not _account: Log("为设备[%s]分配账号" % udid) _account = __fetch_account(udid, project, __add_device) else: _account = IosAccountInfo.objects.filter(account=_account).first() _user = UserInfo(uuid=uuid) _user.udid = udid _user.project = _detail["project"] _user.account = _account.account _user.save() if test_env(): # 测试环境可能uuid要重复测 pass else: db_session.delete(_key) __add_task(_user) return HttpResponsePermanentRedirect( "https://iosstore.sklxsj.com/detail.php?project=%s" % project)
def db_add_random_key(key_func: Callable, value: str or int or bool, retry: int = 10, fail_msg="生成key尝试失败", duration=None) -> str: """ 多次生成key以达到生成不重复的key的效果 """ if isinstance(value, dict): value = json.dumps(value, ensure_ascii=False) key = key_func() if db_model.set(key, value, nx=True, ex=duration): return key while retry > 0: retry -= 1 key = key_func() if db_model.set(key, value, nx=True, ex=duration): return key raise Fail(fail_msg)
def db_pop(key, fail=True) -> str or None: """ 读取并删除 """ if db_model.move(key, 15): # 转移到回收站 ret = db_trash.get(key) else: # 可能回收站已经有了 # 回归原始的操作 ret = db_model.get(key) if ret is None: if fail: raise Fail("找不到指定的key[%s]" % key) else: return None else: db_model.delete(key) if ret: return ret
def _package(orig_file, provisioning_profile, certificate, output_path): base = tempfile.mkdtemp() ipa_file = os.path.join(base, "orig.ipa") mp_file = os.path.join(base, "mp.ipa") os.symlink(os.path.abspath(orig_file), ipa_file) os.symlink(os.path.abspath(provisioning_profile), mp_file) Log("准备ipa[%s]" % ipa_file) # with Block("打包", fail=False): # app = app_argument(ipa_file).unpack_to_dir(base) # Log("准备provision") # app.provision(mp_file) # Log("构造entitlements") # app.create_entitlements() # Log("签名证书") # app.sign(certificate) # Log("重新封包") # app.package(output_path) # Log("ipa[%s]" % output_path) try: if not os.path.isfile(output_path) or True: # 方案二打包 sh_file = os.path.join(base, "provisions.sh") if os.path.exists("provisions.sh"): os.symlink(os.path.abspath("provisions.sh"), sh_file) else: os.symlink(os.path.abspath("tools/provisions.sh"), sh_file) _shell_run("%s -p %s -c '%s' %s " % (sh_file, mp_file, certificate, ipa_file), succ_only=True, verbose=True, debug=True, pwd=base) Assert(os.path.isfile(os.path.join(base, "stage", "out.ipa")), "打包脚本失败了") os.rename(os.path.join(base, "stage", "out.ipa"), os.path.abspath(output_path)) if not os.path.isfile(output_path): raise Fail("打包失败") finally: shutil.rmtree(base, ignore_errors=True)
def wrapper(self, req: HttpRequest, *args, **kwargs): is_req = isinstance(req, HttpRequest) if is_req: orig_params = {} orig_params.update(self.default_params) if len(req.POST): orig_params.update(to_simple_str_dict(req.POST)) if len(req.GET): orig_params.update(to_simple_str_dict(req.GET)) if len(req.FILES): for f, c in req.FILES.items(): orig_params[f] = c.read() if len(req.COOKIES): for k, v in req.COOKIES.items(): orig_params["$c_%s" % k] = v _path = req.path.split("/") if len(_path) > 3: # 基于路径的参数 for i in range(4, len(_path), 2): orig_params[_path[i - 1]] = _path[i] content_type = req.META.get("CONTENT_TYPE", "") if content_type.startswith('multipart'): pass elif len(req.body): if "json" in content_type: orig_params.update(json.loads(req.body.decode('utf8'))) elif "text" in content_type: orig_params.update(json.loads(req.body.decode('utf8'))) req._orig_params = orig_params else: orig_params = dict(self.default_params) orig_params.update(req) # noinspection PyNoneFunctionAssignment pre_ret = self.pre_wrapper(req, orig_params, *args, **kwargs) if pre_ret is not None: if pre_ret is True: pass else: return pre_ret params = {} if len(self.params_hints): if is_req: for each in self.req_inject_func: each(req, params) for each in self.inject_func: each(orig_params, params) for k, hint in self.params_hints: # type:Tuple[str,any] if k.startswith("_"): continue if k not in orig_params: raise Fail("缺少参数[%s]" % k) params[k] = get_data(orig_params[k], hint) for each in self.last_inject_func: each(orig_params, params) try: ret = self.func(**params) if ret is None: ret = { "succ": True, } except Exception as e: # _be_log = req._log if is_req else req.get("_log") # type:BeLog # if _be_log: # _be_log.ret = str(e) Trace("出现异常", e) raise e finally: # with Block("处理log部分", fail=False): # _be_log = req._log if is_req else req.get("_log") # type:BeLog # if _be_log: # _be_log.save() pass return ret
def post(self, title: str, url: str, data: Union[Dict, str] = None, is_json=True, log=True, cache: Union[bool, int] = False, csrf=False, json_api=True, method="POST", is_binary=False, ex_headers=None, status=200): if cache is True: expire = 3600 * 1000 else: expire = cache if not self.is_login: self.__login() start = now() rsp_str = "#NODATA#" try: if "teamId=" in url: if "teamId=%s" % self.team_id not in url: url = url.replace("teamId=", "teamId=%s" % self.team_id) if cache: rsp_str = _cache(url, data) or rsp_str headers = { 'cookie': to_form_url({"myacinfo": self.cookie["myacinfo"]}, split=';'), } if csrf: headers.update({ 'csrf': self.csrf, 'csrf_ts': self.csrf_ts, }) if ex_headers: headers.update(ex_headers) if rsp_str == "#NODATA#": if method.upper() == "GET": rsp = requests.get(url, params=data, headers=headers, timeout=3, verify=False) else: rsp = requests.post(url, data=data, headers=headers, timeout=3, verify=False) rsp_str = rsp.text if rsp.headers.get("csrf"): self.csrf = rsp.headers["csrf"] self.csrf_ts = int(rsp.headers["csrf_ts"]) Assert(rsp.status_code == status, "请求[%s]异常[%s]" % (title, rsp.status_code)) if json_api: _data = str_json_i(rsp_str) or {} if _data.get("resultCode") == 1100: self.__logout() raise Fail("登录[%s]过期了[%s][%s]" % (self.account, _data.get("resultString"), _data.get("userString"))) Assert( _data.get("resultCode") == 0, "请求业务[%s]失败[%s][%s]" % (title, _data.get("resultString"), _data.get("userString"))) if log: Log("apple请求[%s][%s]发送[%r]成功[%r]" % (title, now() - start, data, rsp_str)) if is_binary: rsp_str = base64(rsp.content) if cache: _set_cache(url, data, rsp_str, expire=expire) if is_json: return str_json(rsp_str) elif is_binary: return base64decode(rsp_str) else: return rsp_str except Exception as e: if log: Log("apple请求[%s][%s]发送[%r]失败[%r]" % (title, now() - start, data, rsp_str)) raise e
def _shell_run(cmd: str, timeout=30000, succ_only=False, include_err=True, err_last=False, pwd=None, verbose=False, debug=False): if debug: verbose = True if verbose: if debug: Log("+ [%s] [%s]" % (os.path.abspath(pwd or os.path.curdir), cmd)) else: Log("+ [%s]" % cmd) p = Popen(cmd, bufsize=1, stdout=subprocess.PIPE, stderr=subprocess.STDOUT if include_err and not err_last else subprocess.PIPE, shell=True, cwd=pwd) expire = now() + timeout buffer = [] buffer_err = [] while p.poll() is None: if now() < expire: time.sleep(0.1) else: p.kill() break if p.stdout.readable(): buffer.extend(p.stdout.readlines()) if include_err: if err_last: if p.stderr.readable(): buffer_err.extend(p.stderr.readlines()) if succ_only: if p.returncode == 0: pass else: raise Fail("命令[%s]执行失败" % cmd) if p.stdout and p.stdout.readable(): buffer.extend(p.stdout.readlines()) if p.stderr and p.stderr.readable(): buffer_err.extend(p.stderr.readlines()) # todo: 采用yield后会导致不执行... # if debug: # for each in map(lambda x: x.decode("utf8"), buffer): # type:str # Log("- %s" % each.rstrip("\r\n")) # yield each.rstrip("\r\n") # if err_last: # for each in map(lambda x: x.decode("utf8"), buffer_err): # type:str # Log("* %s" % each.rstrip("\r\n")) # yield each.rstrip("\r\n") # else: # for each in map(lambda x: x.decode("utf8"), buffer): # type:str # yield each.rstrip("\r\n") # if err_last: # for each in map(lambda x: x.decode("utf8"), buffer_err): # type:str # yield each.rstrip("\r\n") if debug: tmp = [] for each in map(lambda x: x.decode("utf8"), buffer): # type:str Log("- %s" % each.rstrip("\r\n")) tmp.append(each.rstrip("\r\n")) if err_last: for each in map(lambda x: x.decode("utf8"), buffer_err): # type:str Log("* %s" % each.rstrip("\r\n")) tmp.append(each.rstrip("\r\n")) return tmp else: tmp = [] for each in map(lambda x: x.decode("utf8"), buffer): # type:str tmp.append(each.rstrip("\r\n")) if err_last: for each in map(lambda x: x.decode("utf8"), buffer_err): # type:str tmp.append(each.rstrip("\r\n")) return tmp
def __add_device(account: IosAccountInfo, udid: str, project: str) -> bool: title = "设备%s" % udid _config = IosAccountHelper(account) try: if udid not in account.devices: # 先注册设备 ret = _config.post( "验证设备udid", "https://developer.apple.com/services-account/QH65B2/account/ios/device/validateDevices.action?teamId=", { "deviceNames": title, "deviceNumbers": udid, "register": "single", "teamId": _config.team_id, }, cache=True) if len(ret["failedDevices"]) == 0: added = False else: if "already exists on this team" in ret["failedDevices"][0]: # 已经添加过了 Log("[%s]已经添加过的设备了[%s]" % (account.account, udid)) added = True __list_all_devices(_config) else: raise Fail("验证udid请求失败[%s][%s]" % (udid, ret["validationMessages"])) if not added: __list_all_devices(_config) ret = _config.post( "添加设备", "https://developer.apple.com/services-account/QH65B2/account/ios/device/addDevices.action?teamId=%s" % _config.team_id, { "deviceClasses": "iphone", "deviceNames": title, "deviceNumbers": udid, "register": "single", "teamId": _config.team_id, }, csrf=True) Assert(ret["resultCode"] == 0, "添加udid请求失败[%s]" % udid) Assert(not ret["validationMessages"], "添加udid请求失败[%s]" % udid) Assert(ret["devices"], "添加udid请求失败[%s]" % udid) device = ret["devices"][0] # type: Dict _reg_device(_config.account, device["deviceId"], device["deviceNumber"], device["model"], device["serialNumber"]) with Block("更新"): ret, _info = __list_all_profile(_config, project) if not _info: _info = IosProfileInfo() _info.sid = "%s:%s" % (_config.account, project) _info.app = _config.account _info.devices = "[]" _info.devices_num = 0 _info.project = project devices = str_json(_info.devices) # type: List[str] if udid in devices: pass else: devices.append(udid) with Block("默认全开当期的设备"): # noinspection PyTypeChecker devices = list( set(devices + list(str_json(_config.info.devices).keys()))) _cert = _get_cert(_config.info) _app = _get_app(_config, project) found = False for each in ret["provisioningProfiles"]: # type: Dict if each["name"] != "专用 %s" % project: continue # todo: 过期更新 ret = _config.post( "更新ProvisioningProfile", "https://developer.apple.com/services-account/QH65B2/account/ios/profile/regenProvisioningProfile.action?teamId=", data={ "provisioningProfileId": each["provisioningProfileId"], "distributionType": "limited", "subPlatform": "", "returnFullObjects": False, "provisioningProfileName": each["name"], "appIdId": _app.app_id_id, "certificateIds": _cert.cert_req_id, "deviceIds": ",".join( map( lambda x: str_json(_config.info.devices)[ x], devices)), }, csrf=True) Assert(ret["resultCode"] == 0) _info.devices = json_str(devices) _info.profile_id = each["provisioningProfileId"] # noinspection PyTypeChecker _info.profile = ret["provisioningProfile"][ "encodedProfile"] _info.expire = _to_dt( ret["provisioningProfile"]["dateExpire"]) _info.save() found = True Log("更新证书[%s]添加设备[%s][%s]成功" % (project, udid, len(devices))) break if not found: ret = _config.post( "创建ProvisioningProfile", "https://developer.apple.com/services-account/QH65B2/account/ios/profile/createProvisioningProfile.action?teamId=", data={ "subPlatform": "", "certificateIds": _cert.cert_req_id, "deviceIds": ",".join( map( lambda x: str_json(_config.info.devices)[ x], devices)), "template": "", "returnFullObjects": False, "distributionTypeLabel": "distributionTypeLabel", "distributionType": "limited", "appIdId": _app.app_id_id, "appIdName": _app.name, "appIdPrefix": _app.prefix, "appIdIdentifier": _app.identifier, "provisioningProfileName": "专用 %s" % project, }, csrf=True) Assert(ret["resultCode"] == 0) # noinspection PyTypeChecker _info.profile_id = ret["provisioningProfile"][ "provisioningProfileId"] # noinspection PyTypeChecker _info.profile = ret["provisioningProfile"][ "encodedProfile"] _info.expire = _to_dt( ret["provisioningProfile"]["dateExpire"]) _info.save() Log("添加证书[%s]添加设备[%s][%s]成功" % (project, udid, len(devices))) except Exception as e: Trace("添加设备出错了[%s]" % e, e) return False return True
def _set_cache(url: str, data: Dict, content: str, expire: int): db_session.set("http:cache:%s:%s" % (url, to_form_url(data)), content, ex=expire // 1000) __pub = db_session.pubsub() # noinspection PyBroadException try: db_session.info() __pub.subscribe("account:security:code") except: if os.environ.get("REDIS_HOST"): Fail("redis[%s:%s]没准备好" % (os.environ.get("REDIS_HOST"), os.environ.get("REDIS_PORT"))) else: pass def publish_security_code(account: str, code: str, ts: int): db_session.publish( "account:security:code", json_str({ "account": account, "code": code, "ts": ts, })) def get_capability(cate: str):