def service(): # 描述性分析 db获取任务 list_ask_queue = getDescribeAnalysiscList() print "DescribeAnalysis, ", len(list_ask_queue) if list_ask_queue.__len__() <= 0: return for ask_parm in list_ask_queue: print ask_parm dict_id_str = ask_parm["dict_list"] dict_id_Arrs = dict_id_str.split(',') filterList = [] for dict_id in dict_id_Arrs: filterList.append("_" + dict_id) # 描述性分析 db获取分组 list_group_queue = getDescribeAnalysisDBList(ask_parm['groupID']) if list_group_queue.__len__() < 1: print "warning:", list_group_queue.__len__() continue # 描述性分析 获取计算 result_dict = calc_python_spark(ask_parm, filterList, list_group_queue[0]) print " result_dict", result_dict uuId = ask_parm["uuId"] crowID = ask_parm["crowID"] utype = ask_parm["type"] print result_dict if result_dict is None: print crowID, uuId updateDescribeAnalysisStatus(uuId) continue print "utype--", utype for dict_id in dict_id_Arrs: try: if (utype == "0"): saveDescribeAnalysisResultToDB( utils.create_id(uuId + crowID + dict_id), uuId, dict_id, result_dict["_" + dict_id]) updateDescribeAnalysisStatus(uuId) else: saveDescribeAnalysisResultToDB( utils.create_id(uuId + crowID + dict_id + "0"), uuId, dict_id, result_dict["_" + dict_id][0]) saveDescribeAnalysisResultToDB( utils.create_id(uuId + crowID + dict_id + "1"), uuId, dict_id, result_dict["_" + dict_id][1]) updateDescribeAnalysisStatus(uuId) except ZeroDivisionError, e: print e.message print "done", uuId, crowID, dict_id
def search(self, results, media, lang, manual): PlexLog.debug("=================== Search Start ===================") PlexLog.debug("%s (%s)" % (self.name, self.ver)) PlexLog.debug("Plex version: %s" % Platform.ServerVersion) server = Prefs["Server"] if not server: PlexLog.error("Missing server!") return movie = request_json(urljoin(server, "movie"), as_movie(media)) if movie is None: return title = movie.get("title") if title is None: PlexLog.error("Missing or invalid title: %s" % str(movie)) return aired = convert_date(movie.get("aired")) year = aired.year if aired is not None else 0 # Plex throws exception that have "/" in ID mid = create_id(title, year) result = MetadataSearchResult(id=mid, name=title, year=year, lang=lang, score=100) results.Append(result) PlexLog.debug("=================== Search end ===================")
def search(self, results, media, lang, manual): PlexLog.debug("=================== Search Start ===================") PlexLog.debug("%s (%s)" % (self.name, self.ver)) PlexLog.debug("Plex version: %s" % Platform.ServerVersion) show = get_show(media) if show is None: return title = show.get("title") if title is None: PlexLog.error("Missing or invalid title: %s" % str(show)) return aired = convert_date(show.get("aired")) year = aired.year if aired is not None else 0 # Plex throws exception that have "/" in ID mid = create_id(title, year) result = MetadataSearchResult(id=mid, name=title, year=year, lang=lang, score=100) results.Append(result) PlexLog.debug("=================== Search end ===================")
def service(): list_ask_queue = getSampleDBAskCalcList() print "SampleTest, ", len(list_ask_queue) if list_ask_queue.__len__() <= 0: return for ask_parm in list_ask_queue: print ask_parm ask_parm['popmean'] = float(ask_parm['popmean']); ask_parm['confidence'] = float(ask_parm['confidence']); dic_id_str = ask_parm["dic_id"]; dic_id_Arrs = dic_id_str.split(',') filterList = [] for dic_id in dic_id_Arrs: filterList.append("_"+dic_id); print ask_parm result_dict = calc_python_spark(ask_parm, filterList) uuId = ask_parm["uuId"] crowID = ask_parm["crowID"] print result_dict for dic_id in dic_id_Arrs : if result_dict.__contains__("_"+dic_id) : result = result_dict["_"+dic_id] else : result = None saveSampleResultToDB(utils.create_id(uuId+crowID+dic_id), uuId, dic_id, result) updateSampleStatus(uuId) return
def serverHandler(): while True : list_ask_queue = getStudentTTestDBAskCalcList() if list_ask_queue.__len__() <= 0: continue for ask_parm in list_ask_queue: print ask_parm ask_parm['confidence'] = float(ask_parm['confidence']); dict_id_str = ask_parm["dict_list"]; dict_id_Arrs = dict_id_str.split(',') filterList = [] for dict_id in dict_id_Arrs: filterList.append("_"+dict_id); list_group_queue = getStudentTTestGroupDBList(ask_parm['s_group_id1'], ask_parm['s_group_id2']) if list_group_queue.__len__() < 2 : print "warning:", list_group_queue.__len__() continue result_dict = calc_python_spark(ask_parm, filterList, list_group_queue); print result_dict uuId = ask_parm["uuId"] crowID = ask_parm["crowID"] for dict_id in dict_id_Arrs: saveStudentTtestResultToDB(utils.create_id(uuId+crowID+dict_id), uuId, dict_id, result_dict["_"+dict_id]) updateStudentTtestStatus(uuId) time.sleep(10000) return
def get_entities(persons): entities = [] for person in persons: name = person.pop(PER_NAME) values = person.values() unique_id = create_id([_.encode('utf-8') for _ in values]) fields = [ {'tag': t, 'value': v} for t, v in person.items() ] entities.append(create_entity(unique_id, 'person', name, fields)) return entities
def get_entities(persons): entities = [] for person in persons: name = person.pop('person_name') values = person.values() unique_id = create_id([_.encode('utf-8') for _ in values]) fields = [ {'tag': t, 'value': WHITESPACE_PATTERN.sub('', v)} for t, v in person.items() ] entities.append(create_entity(unique_id, 'person', name, fields)) return entities
def get_entities(persons): entities = [] for person in persons: name = person[PER_NAME] values = person.values() unique_id = create_id([_.encode('utf-8') for _ in values]) fields = [ {'tag': t.strip('!'), 'value': v} for t, v in person.items() ] entities.append(create_entity(unique_id, 'person', name, fields)) return entities
def __init__(self, variable_list: list): """ :complexity: O(n) where n is the number of variables in the clause :param variable_list: list of variable """ if isinstance(variable_list, set): variable_list = [Variable(var) for var in variable_list] self.variable_list = variable_list self.__size = len(self.variable_list) self.id = utils.create_id() self.literals_set = set() for var in variable_list: self.literals_set.add(var.variable_value) self.__tautology = self.__setup_tautology()
def search(self, results, media, lang, manual): PlexLog.debug("=================== Search Start ===================") PlexLog.debug("%s (%s)" % (self.name, self.ver)) PlexLog.debug("Plex version: %s" % Platform.ServerVersion) server = Prefs["Server"] authKey = Prefs["AuthKey"] if not server: PlexLog.error("Missing server!") return requestUrl = urljoin(server, "show") if authKey: requestUrl = requestUrl + "?AuthKey=" + authKey PlexLog.debug("Requesting URL: %s" % requestUrl) show = request_json(requestUrl, as_show(media)) if show is None: return title = show.get("title") if title is None: PlexLog.error("Missing or invalid title: %s" % str(show)) return aired = convert_date(show.get("aired")) year = aired.year if aired is not None else 0 # Plex throws exception that have "/" in ID mid = create_id(title, year) result = MetadataSearchResult(id=mid, name=title, year=year, lang=lang, score=100) results.Append(result) PlexLog.debug("=================== Search end ===================")
def post(self, *args, **kwargs): if 'user' not in kwargs or args: self.raise401() grant_type = self.get_argument('grant_type', None) response_type = self.get_argument('response_type', None) redirect_uris = self.get_argument('redirect_uris', None) app_name = self.get_argument('app_name', None) description = self.get_argument('description', None) website = self.get_argument('website', None) try: user = kwargs['user'] client_id = create_id() client_secret = create_secret() grant_type = grant_type or 'authorization_code' response_type = response_type or 'code' # todo scopes default_scopes = ['tasks', 'projects', 'repos', 'users', 'teams'] scopes = default_scopes redirect_uris = parse_listed_strs(redirect_uris) # todo default default_redirect_uri = redirect_uris[0] if redirect_uris else '' client = Client( client_id=client_id, client_secret=client_secret, user=user, grant_type=grant_type, response_type=response_type, scopes=scopes, default_scopes=default_scopes, redirect_uris=redirect_uris, default_redirect_uri=default_redirect_uri, website=website, app_name=app_name, description=description) client.save() client_data = document_to_json(client, filter_set=_FILTER) self.set_status(201) self.write(client_data) except Exception as e: reason = e.message self.raise400(reason=reason)
def serverHandler(): list_ask_queue = getSampleDBAskCalcList() for ask_parm in list_ask_queue: print ask_parm ask_parm['popmean'] = float(ask_parm['popmean']) ask_parm['confidence'] = float(ask_parm['confidence']) dic_id_str = ask_parm["dic_id"] dic_id_Arrs = dic_id_str.split(',') filterList = [] for dic_id in dic_id_Arrs: filterList.append("_" + dic_id) print ask_parm result_dict = calc_python_spark(ask_parm, filterList) uuId = ask_parm["uuId"] crowID = ask_parm["crowID"] print result_dict for dic_id in dic_id_Arrs: saveSampleResultToDB(utils.create_id(uuId + crowID + dic_id), uuId, dic_id, result_dict["_" + dic_id]) updateSampleStatus(uuId)
def service(): # 获取db ,任务队列 list_ask_queue = getStudentTTestDBAskCalcList() print "StudentTtest, ", len(list_ask_queue) if list_ask_queue.__len__() <= 0: return for ask_parm in list_ask_queue: print ask_parm ask_parm['confidence'] = float(ask_parm['confidence']) dict_id_str = ask_parm["dict_list"] dict_id_Arrs = dict_id_str.split(',') filterList = [] for dict_id in dict_id_Arrs: filterList.append("_" + dict_id) # 获取分组 list_group_queue = getStudentTTestGroupDBList(ask_parm['s_group_id1'], ask_parm['s_group_id2']) if list_group_queue.__len__() < 2: print "warning:", list_group_queue.__len__() continue # spark 计算分组 result_dict = calc_python_spark(ask_parm, filterList, list_group_queue) print result_dict uuId = ask_parm["uuId"] crowID = ask_parm["crowID"] for dict_id in dict_id_Arrs: # 计算结果保存与更新 saveStudentTtestResultToDB( utils.create_id(uuId + crowID + dict_id), uuId, dict_id, result_dict["_" + dict_id]) updateStudentTtestStatus(uuId) return
def _get_token(request): token = request.COOKIES.get("token", None) if not token: token = utils.create_id() print 'created token', token return token
opt = parser.parse_args() DATA_NAME, RECALLS, BATCH_SIZE, NUM_EPOCHS = opt.data_name, opt.recalls, opt.batch_size, opt.num_epochs ENSEMBLE_SIZE, META_CLASS_SIZE = opt.ensemble_size, opt.meta_class_size recall_ids = [int(k) for k in RECALLS.split(',')] DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu') data_dicts = torch.load('data/{}/data_dicts.pth'.format(DATA_NAME)) train_data, test_data = data_dicts['train'], data_dicts['test'] # sort classes and fix the class order all_class = sorted(train_data) idx_to_class = {i: all_class[i] for i in range(len(all_class))} for i in range(1, ENSEMBLE_SIZE + 1): print('Training ensemble #{}'.format(i)) meta_id = create_id(META_CLASS_SIZE, len(data_dicts['train'])) meta_data_dict = load_data(meta_id, idx_to_class, train_data) model = Model(META_CLASS_SIZE).to(DEVICE) optimizer = Adam(model.parameters(), lr=1e-4) lr_scheduler = MultiStepLR( optimizer, milestones=[int(NUM_EPOCHS * 0.5), int(NUM_EPOCHS * 0.7)], gamma=0.1) criterion = CrossEntropyLoss() best_acc, best_model = 0, None for epoch in range(1, NUM_EPOCHS + 1): lr_scheduler.step(epoch) train_loss, train_acc = train(model, meta_data_dict, optimizer) print('Epoch {}/{} - Loss:{:.4f} - Acc:{:.4f}'.format(