def joinData(cls, messages, userId=None): single = False if type(messages) != list: single = True messages = [messages] ids = [message["_id"] for message in messages] userIds = [message["fromId"] for message in messages] users = core.fetch(keys=userIds) if userId: readStatuses = core.fetch(db=core.connect("shiftspace/shared"), keys=[Message.makeReadId(id, userId) for id in ids]) for i in range(len(messages)): if (userIds[i]): if (userIds[i] == 'shiftspace'): messages[i]["gravatar"] = "images/default.png" messages[i]["userName"] = '******' else: messages[i]["gravatar"] = (users[i]["gravatar"] or "images/default.png") messages[i]["userName"] = users[i]["userName"] messages[i]["modifiedStr"] = utils.pretty_date(utils.futcstr(messages[i]["modified"])) if userId: messages[i]["read"] = (readStatuses[i] != None) if single: return messages[0] else: return messages
def joinData(cls, comments): single = False if type(comments) != list: single = True comments = [comments] authorIds = [comment["shiftAuthor"] for comment in comments] shiftIds = [comment["shiftId"] for comment in comments] userIds = [comment["createdBy"] for comment in comments] users = core.fetch(keys=userIds) authors = core.fetch(keys=authorIds) shifts = core.fetch(core.connect("shiftspace/shared"), keys=shiftIds) commentCounts = core.fetch(core.connect("shiftspace/shared"), view=Comment.count_by_shift, keys=shiftIds) for i in range(len(comments)): if (authors[i]): comments[i]["authorName"] = authors[i]["userName"] comments[i]["userName"] = users[i]["userName"] comments[i]["gravatar"] = users[i]["gravatar"] comments[i]["createdStr"] = utils.pretty_date(utils.futcstr(comments[i]["created"])) comments[i]["commentCount"] = commentCounts[i] comments[i]["space"] = shifts[i]["space"] comments[i]["href"] = shifts[i]["href"] comments[i]["domain"] = shifts[i]["domain"] if single: return comments[0] else: return comments
def joinData(cls, shifts, userId=None): from server.models.favorite import Favorite from server.models.comment import Comment if core._local: db = core.conect("user/%s" % core._local_id) else: db = core.connect("shiftspace/shared") single = False if type(shifts) != list: single = True shifts = [shifts] ids = [shift['_id'] for shift in shifts] favIds = [Favorite.makeId(userId, shiftId) for shiftId in ids] isFavorited = core.fetch(db, keys=favIds) favCounts = core.fetch(db, view=Favorite.count_by_shift, keys=ids) commentCounts = core.fetch(db, view=Comment.count_by_shift, keys=ids) userIds = [shift["createdBy"] for shift in shifts] users = core.fetch(keys=userIds) for i in range(len(shifts)): shifts[i]["favorite"] = (isFavorited[i] != None) shifts[i]["favoriteCount"] = favCounts[i] shifts[i]["commentCount"] = commentCounts[i] shifts[i]["gravatar"] = (users[i]["gravatar"] or "images/default.png") shifts[i]["userName"] = users[i]["userName"] shifts[i]["createdStr"] = utils.pretty_date(utils.futcstr(shifts[i]["created"])) shifts[i]["modifiedStr"] = utils.pretty_date(utils.futcstr(shifts[i]["modified"])) if single: return shifts[0] else: return shifts
def shareWith(self, userIds, fromUser=None): from server.models.message import Message from server.models.ssuser import SSUser users = core.fetch(keys=userIds) if fromUser: userName = fromUser.userName else: userName = SSUser.read(self.createdBy).userName for user in users: json = { "fromId": self.createdBy, "toId": user["_id"], "title": "%s has shared a shift with you!" % userName, "text": "%s has shared a shift titled '%s' with you!" % (userName, self.summary), "meta": "share", "content": { "type": "shift", "_id": self.id, "href": self.href, "summary": self.summary } } Message.create(**json)
def joinData(cls, permissions, userId=None): from server.models.group import Group groupIds = [permission.groupId for permission in permissions] groups = core.fetch(keys=groupIds) for i in range(len(permissions)): permissions[i]["shortName"] = groups[i]["shortName"] permissions[i]["longName"] = groups[i]["longName"] return permissions
def followers(self, start=None, end=None, limit=25): from server.models.follow import Follow if not start: start = [self.id] if not end: end = [self.id, {}] results = Follow.followers_by_created(core.connect(), limit=limit) userIds = core.values(results[start:end]) return core.fetch(keys=userIds)
def joinData(cls, groups, userId=None): from server.models.permission import Permission db = core.connect() keys = [[userId, group.id] for group in groups] perms = core.fetch(db, view=Permission.by_user_and_group, keys=keys) for i in range(len(groups)): if perms[i]: groups[i]["level"] = perms[i]["level"] return groups
def favorites(self, start=None, end=None, limit=25): from server.models.favorite import Favorite db = core.connect("shiftspace/shared") if not start: start = [self.id] if not end: end = [self.id, {}] results = Favorite.by_user_and_created(db, limit=limit) favs = core.objects(results[start:end]) return core.fetch(db, keys=[fav.shiftId for fav in favs])
def joinData(cls, permissions, userId=None): from server.models.group import Group groupIds = [permission.groupId for permission in permissions] groups = core.fetch(keys=groupIds) for i in range(len(permissions)): permissions[i]["shortName"] = groups[i]["shortName"] permissions[i]["longName"] = groups[i]["longName"] permissions[i]["tagLine"] = groups[i]["tagLine"] permissions[i]["url"] = groups[i]["url"] return permissions
def list(avitem): url_page = 'http://www.youku.com/show_page/id_' + avitem['id'] page_page = core.fetch(url_page).decode('utf-8') ul_reloads = re.search(r'<ul id="zySeriesTab">([\s\S]*?)</ul>', page_page).group(1) reloads = re.finditer(r'<li data="(.*?)"[\s\S]*?>', ul_reloads) ret = [] for reload in reloads: url_episodes = 'http://www.youku.com/show_episode/id_' + avitem['id'] + '?divid=' + reload.group(1) page_episodes = core.fetch(url_episodes).decode('utf-8') episodes = re.finditer(r'<ul[\s\S]*?>\s*?<li[\s\S]*?>[\s\S]*?<a([\s\S]*?charset=".*?"[\s\S]*?)>[\s\S]*?</a>[\s\S]*?</li>', page_episodes) for episode in episodes: attrs = episode.group(1) title_episode = re.search(r'title="([\s\S]*?)"', attrs).group(1) this = {'title': title_episode} core.apply_default(this, avitem) url_episode = re.search(r'href="(.+?)"', attrs).group(1) this['url'] = url_episode ret.append(this) return ret
def getIdByPage(url): header = core.fetch_header(url) if re.match(r'text/html', header['content-type']): page = core.fetch(url).decode('utf-8') regexps = [ r'<head>[\s\S]*?var videoId\s*=\s*\'(.*)\'[\s\S]*?</head>', r'<head>[\s\S]*?var videoId2\s*=\s*\'(.*)\'[\s\S]*?</head>' ] return core.regexps_search(regexps, page) else: return False
def joinData(cls, shifts, userId=None): from server.models.favorite import Favorite from server.models.comment import Comment if core._local: db = core.conect("user/%s" % core._local_id) else: db = core.connect("shiftspace/shared") single = False if type(shifts) != list: single = True shifts = [shifts] ids = [shift['_id'] for shift in shifts] favIds = [Favorite.makeId(userId, shiftId) for shiftId in ids] isFavorited = core.fetch(db, keys=favIds) favCounts = core.fetch(db, view=Favorite.count_by_shift, keys=ids) commentCounts = core.fetch(db, view=Comment.count_by_shift, keys=ids) userIds = [shift["createdBy"] for shift in shifts] users = core.fetch(keys=userIds) for i in range(len(shifts)): shifts[i]["favorite"] = (isFavorited[i] != None) shifts[i]["favoriteCount"] = favCounts[i] shifts[i]["commentCount"] = commentCounts[i] shifts[i]["gravatar"] = (users[i]["gravatar"] or "images/default.png") shifts[i]["userName"] = users[i]["userName"] shifts[i]["createdStr"] = utils.pretty_date( utils.futcstr(shifts[i]["created"])) shifts[i]["modifiedStr"] = utils.pretty_date( utils.futcstr(shifts[i]["modified"])) if single: return shifts[0] else: return shifts
def get(self): url = ( 'http://openexchangerates.org/api/latest.json?app_id=' + keys.openexchangerates ) data = memcache.get(url) if data is None: result = core.fetch(url) if result: data = result.content memcache.set(url, data, time=43200) else: return self.error(500) self.response.headers['Content-Type'] = 'application/json' self.response.out.write(data)
def get(self, viewType): # fetch TSV data from Bricklink.com TYPES = ['parts', None, None, 'colors', None, 'codes'] viewType = viewType.lower() if viewType not in TYPES: return self.error(404) num = TYPES.index(viewType) result = core.fetch( 'http://www.bricklink.com/catalogDownload.asp' + '?a=a&itemType=P&downloadType=T&viewType=' + str(num) ) if result: core.dbSaveText(viewType, result.content) self.response.out.write('succeeded') else: self.error(500)
def list(avitem): url_playlist = 'http://www.youku.com/playlist_show/id_' + avitem['id'] + '_ascending_1_mode_1' ret = [] while url_playlist: page_playlist = core.fetch(url_playlist).decode('utf-8') div_list = re.search(r'<div class="pack_list">([\s\S]*?)</div>', page_playlist).group(1) matches = re.finditer(r'<a([\s\S]*?)>[\s\S]*?</a>', div_list) for match in matches: attr = match.group(1) if attr.find('target="video"') != -1: title = re.search(r'title="([\s\S]*?)"', attr).group(1) url = re.search(r'href="([\s\S]*?)"', attr).group(1) this = {'title': title, 'url': url} core.apply_default(this, avitem) ret.append(this) div_pages = re.search(r'<div class="page f_r">([\s\S]*?)</div>', page_playlist).group(1) url_playlist = core.regexps_search([ur'<a href="(.+)" title=\'下一页\'>>></a>'], div_pages) return ret
def users(cls, start=None, end=None, limit=25, groupId=None): results = User.all_by_joined(core.connect(), limit=25) if start and not end: users = core.objects(results[start:]) elif not start and end: users = core.objects(results[:end]) elif start and end: users = core.objects(results[start:end]) else: users = core.objects(results) if groupId: from server.models.permission import Permission keys = [[user.id, groupId] for user in users] isMembers = core.fetch(core.connect(), view=Permission.is_member, keys=keys) for i in range(len(users)): if isMembers[i]: users[i]["member"] = True else: users[i]["member"] = False return users
def fetchSuggestions(self, callback=None, args=(), schedule="now"): core.fetch("youtube", "search", {"relatedToVideoId": self.id}, self.setSuggestions, (callback, args), schedule)
def fetchThumbnail(self, callback=None, args=(), schedule="now"): core.fetch("youtube", "content", {"url": self.thumbnailUrl}, self.setThumbnail, (callback, args), schedule)
def namesToIds(cls, names): return [ user["_id"] for user in core.fetch(view=SSUser.by_name, keys=names) if user and user.get("_id") ]
def shortNamesToIds(cls, shortNames): return [group["_id"] for group in core.fetch(view=Group.by_short_name, keys=shortNames)]
def members(self): from server.models.permission import Permission db = core.connect() return core.fetch(core.connect(), keys=[row.value for row in Permission.all_members(db, key=self.id).rows])
def __getId_page(url): page = core.fetch(url).decode('utf-8') return core.regexps_search([r'http://v.youku.com/v_show/id_[A-Za-z0-9=]+\.html\?f=(\d+)'], page)
def fetchStreamUrl(self, callback=None, args=(), schedule="now"): core.fetch("youtube", "stream", {"id": self.id}, self.setStreamUrl, (callback, args), schedule)
def namesToIds(cls, names): return [user["_id"] for user in core.fetch(view=SSUser.by_name, keys=names) if user and user.get("_id")]
def resolve(avitem): import json format = {'flv': 'flv', 'mp4': 'mp4', 'hd2': 'flv', '3gphd': 'mp4', '3gp': 'flv', 'hd3': 'flv'} quality = {'flv': '0', 'flvhd': '0', 'mp4': '1', 'hd2': '2', '3gphd': '1', '3gp': '0', 'hd3': '3'} core.apply_default(avitem, {'quality': ['flv', 'flvhd', 'mp4', 'hd2', 'hd3', '3gphd', '3gp'], 'path': '%title%%part%.%ext%'}) url_filelist = 'http://v.youku.com/player/getPlayList/VideoIDS/' + avitem['id'] + '/Pf/4?password='******'password'): url_filelist += avitem['password'] filelist = json.loads(core.fetch(url_filelist).decode('utf-8'))['data'][0] if filelist.has_key('error_code'): return 'ERROR' # !!ERROR!! avitem['title'] = filelist['title'] for seg in avitem['quality']: if filelist['segs'].has_key(seg): charset = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ/\\:._-1234567890' cg = '' seed = filelist['seed'] while len(charset) > 0: seed = (211 * seed + 30031) % 65536 idx = seed * len(charset) / 65536 cg += charset[idx] charset = charset[:idx] + charset[(idx+1):] cg_fun = '' for i in filelist['streamfileids'][seg].split('*')[:-1]: cg_fun += cg[int(i)] files = [] for part in filelist['segs'][seg]: f = cg_fun[:8] + '%02X' % int(part['no']) + cg_fun[10:] a = part['k'] if a == "" or a == -1: a = filelist['key2'] + filelist['key1'] url_part = 'http://f.youku.com/player/getFlvPath/sid/00_00/st/' + format[seg] + '/field/' + f + '?K=' + a + '&hd=' + quality[seg] size_part = int(part['size']) this = { 'url': url_part, 'size': size_part, 'path': core.getPath(avitem['path'], { 'quality': seg, 'title': avitem['title'], 'id': avitem['id'], 'ext': format[seg], 'part': '%03d' % int(part['no']) }) } files.append(this) if core.fetch_error(files[0]['url']): core.output('warning: resolved url not vaild. try next format...', 2) else: avitem['files'] = files avitem['path'] = core.getPath(avitem['path'], { 'quality': seg, 'title': avitem['title'], 'id': avitem['id'], 'ext': format[seg], 'part': '' }) break if not avitem.has_key('files'): core.output('error: failed resolving url.') # !! error !! return avitem
def getIdByPage(url): page = core.fetch(url).decode('utf-8') return core.regexps_search([r'<head>[\s\S]*window.g_config={\s*id:\'([a-z0-9]+)\'[\s\S]*</head>'], page)