def main(): con = rd.get_connection() db = mongo.get_db() keywords = [] for topic in db.topic.find({'status': RECORD_STATUS.NORMAL}): keywords.append(topic['main_key']) keywords.extend(topic['synonyms']) #self.keywords = [keyword.encode('utf-8') for keyword in keywords] conn = psycopg2.connect(user="******", database="sandbox_pg16_buzz", host="116.213.213.117", password="******") cursor = conn.get_cursor() for keyword in keywords: try: table_name = u"%s_20130601" sql = "select url from %s " % table_name + " order by article_id;" cursor.execute(sql) for row in cursor.fetchall(): url, = row print url item = { "url" : url, "isseed" : False, "isindex": False, "timestamp": time.time()} domain = get_domain(row['url']) con.lpush(domain, pickle_dumps(item)) except e: print e
def ganga(self, line, cell): # args = parse_argstring(self.ganga, line) # if args.save: # # file_name = str(args.save) # # code = "" # # if cell is None: # # code = line # # else: # # code = cell # # with open(file_name, 'w') as f: # # f.write(code) # pass # else: # logger.info("Magic Cell Execution starts") #monitor.send({"msgtype": "magic_execution_start"}) code = "" if cell is None: code = line else: code = cell # logger.info("Sending Code to Ganga from kernel") # Start execution current_ns = pickle_dumps(self._pickle_ns()) op = monitor.run(code, (current_ns)) if op[0].stderr: print(op[0].stderr) if op[0].stdout: print(op[0].stdout)
def __wrapper(*args, **kwargs): key = md5( pickle_dumps((func.__name__, args, kwargs)) ).hexdigest() if key in CACHE and not is_expired(CACHE[key]['time'], cache_time): return CACHE[key]['data'] CACHE[key] = {'time': time(), 'data': func()} return CACHE[key]['data']
def regex(): try: compiled = re.compile(request.form.get('regex')) except re.error as e: db.users.update({'_id': bson.ObjectId(session['user_id'])}, {'$set': {'regex': request.form.get('regex')}}) flash('Invalid regular expression: %s' % e, 'danger') else: pickled = pickle_dumps(compiled) db.users.update({'_id': bson.ObjectId(session['user_id'])}, {'$set': {'regex': pickled}}) return redirect(url_for('profile'))
def _pickle_ns(self): new_ns = dict() for k, v in self.shell.user_ns.items(): if not k in DEFAULT_BLACKLIST: try: if ismodule(v): pass else: _ = pickle_dumps({k: v}) new_ns[k] = v except PicklingError: continue except Exception: continue return new_ns
def _handle_request_call(self, func, args, kw, dtype, index, obj_id, name, argkw): try: let = getcurrent() setattr(let, _service_name_, self) if args is None: rs = func() else: rs = func(*args, **kw) if kw is not None else func(*args) if dtype & ST_NO_RESULT: return if not getattr(func, '_rpc_pickle_result_', False): self.send(RT_RESPONSE, index, dumps(rs)) else: self.send(RT_RESPONSE | DT_PICKLE, index, pickle_dumps(rs, PICKLE_PROTOCOL)) except Exception as e: log_except('export(%s).%s(%s)', obj_id, name, repr(argkw)) if dtype & ST_NO_RESULT or self.svr.stoped: return self.send(RT_EXCEPTION, index, str(e))
def call(self, obj_id, name, args, kw, no_result=False, timeout=CALL_TIMEORUT, pickle=False, proxy=False): dtype = RT_REQUEST if proxy: objs = args[0] #first arg is proxy(str, RpcProxy or list) if isinstance(objs, (tuple, list)): obj_ids = [] for o in objs: obj, is_pickle = self._reg_obj(o) pickle = pickle or is_pickle obj_ids.append(obj) else: obj, is_pickle = self._reg_obj(objs) pickle = pickle or is_pickle obj_ids = obj args = (obj_ids, ) + args[1:] dtype |= DT_PROXY if pickle: dtype |= DT_PICKLE argkw = pickle_dumps((args, kw), PICKLE_PROTOCOL) else: argkw = dumps((args, kw)) if len(argkw) >= ZIP_LENGTH: dtype |= DT_ZIP argkw = zlib.compress(argkw, ZIP_LEVEL) if no_result: dtype |= ST_NO_RESULT index = self.next_id() #iter_id.next() self.send(dtype, obj_id, index, name, argkw) if no_result: return result = self._read_response(index, timeout) return result
def encode_label(args): return base64_encodestring(pickle_dumps(args)).strip()
def deck_insert (deck): CONN.execute("""INSERT OR REPLACE INTO decks VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", (deck.id, deck.name, deck.author, deck.url, deck.type, deck.hero_class, deck.dust_cost, deck.rating, deck.num_view, deck.num_comment, int(time.mktime(deck.time_update)), buffer(pickle_dumps(deck.cards, -1)), deck.scan_count))