def main():
con = rd.get_connection()
db = mongo.get_db()
keywords = []
for topic in db.topic.find({'status': RECORD_STATUS.NORMAL}):
keywords.append(topic['main_key'])
keywords.extend(topic['synonyms'])
#self.keywords = [keyword.encode('utf-8') for keyword in keywords]
conn = psycopg2.connect(user="******", database="sandbox_pg16_buzz", host="116.213.213.117", password="******")
cursor = conn.get_cursor()
for keyword in keywords:
try:
table_name = u"%s_20130601"
sql = "select url from %s " % table_name + " order by article_id;"
cursor.execute(sql)
for row in cursor.fetchall():
url, = row
print url
item = {
"url" : url,
"isseed" : False,
"isindex": False,
"timestamp": time.time()}
domain = get_domain(row['url'])
con.lpush(domain, pickle_dumps(item))
except e:
print e
def ganga(self, line, cell):
# args = parse_argstring(self.ganga, line)
# if args.save:
# # file_name = str(args.save)
# # code = ""
# # if cell is None:
# # code = line
# # else:
# # code = cell
# # with open(file_name, 'w') as f:
# # f.write(code)
# pass
# else:
# logger.info("Magic Cell Execution starts")
#monitor.send({"msgtype": "magic_execution_start"})
code = ""
if cell is None:
code = line
else:
code = cell
# logger.info("Sending Code to Ganga from kernel")
# Start execution
current_ns = pickle_dumps(self._pickle_ns())
op = monitor.run(code, (current_ns))
if op[0].stderr:
print(op[0].stderr)
if op[0].stdout:
print(op[0].stdout)
def __wrapper(*args, **kwargs):
key = md5(
pickle_dumps((func.__name__, args, kwargs))
).hexdigest()
if key in CACHE and not is_expired(CACHE[key]['time'], cache_time):
return CACHE[key]['data']
CACHE[key] = {'time': time(), 'data': func()}
return CACHE[key]['data']
def regex():
try:
compiled = re.compile(request.form.get('regex'))
except re.error as e:
db.users.update({'_id': bson.ObjectId(session['user_id'])},
{'$set': {'regex': request.form.get('regex')}})
flash('Invalid regular expression: %s' % e, 'danger')
else:
pickled = pickle_dumps(compiled)
db.users.update({'_id': bson.ObjectId(session['user_id'])},
{'$set': {'regex': pickled}})
return redirect(url_for('profile'))
def _pickle_ns(self):
new_ns = dict()
for k, v in self.shell.user_ns.items():
if not k in DEFAULT_BLACKLIST:
try:
if ismodule(v):
pass
else:
_ = pickle_dumps({k: v})
new_ns[k] = v
except PicklingError:
continue
except Exception:
continue
return new_ns
def _handle_request_call(self, func, args, kw, dtype, index, obj_id, name,
argkw):
try:
let = getcurrent()
setattr(let, _service_name_, self)
if args is None:
rs = func()
else:
rs = func(*args, **kw) if kw is not None else func(*args)
if dtype & ST_NO_RESULT:
return
if not getattr(func, '_rpc_pickle_result_', False):
self.send(RT_RESPONSE, index, dumps(rs))
else:
self.send(RT_RESPONSE | DT_PICKLE, index,
pickle_dumps(rs, PICKLE_PROTOCOL))
except Exception as e:
log_except('export(%s).%s(%s)', obj_id, name, repr(argkw))
if dtype & ST_NO_RESULT or self.svr.stoped:
return
self.send(RT_EXCEPTION, index, str(e))
def call(self,
obj_id,
name,
args,
kw,
no_result=False,
timeout=CALL_TIMEORUT,
pickle=False,
proxy=False):
dtype = RT_REQUEST
if proxy:
objs = args[0] #first arg is proxy(str, RpcProxy or list)
if isinstance(objs, (tuple, list)):
obj_ids = []
for o in objs:
obj, is_pickle = self._reg_obj(o)
pickle = pickle or is_pickle
obj_ids.append(obj)
else:
obj, is_pickle = self._reg_obj(objs)
pickle = pickle or is_pickle
obj_ids = obj
args = (obj_ids, ) + args[1:]
dtype |= DT_PROXY
if pickle:
dtype |= DT_PICKLE
argkw = pickle_dumps((args, kw), PICKLE_PROTOCOL)
else:
argkw = dumps((args, kw))
if len(argkw) >= ZIP_LENGTH:
dtype |= DT_ZIP
argkw = zlib.compress(argkw, ZIP_LEVEL)
if no_result:
dtype |= ST_NO_RESULT
index = self.next_id() #iter_id.next()
self.send(dtype, obj_id, index, name, argkw)
if no_result:
return
result = self._read_response(index, timeout)
return result
def encode_label(args):
return base64_encodestring(pickle_dumps(args)).strip()
def encode_label(args):
return base64_encodestring(pickle_dumps(args)).strip()
def deck_insert (deck):
CONN.execute("""INSERT OR REPLACE INTO decks VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", (deck.id, deck.name, deck.author, deck.url, deck.type, deck.hero_class, deck.dust_cost, deck.rating, deck.num_view, deck.num_comment, int(time.mktime(deck.time_update)), buffer(pickle_dumps(deck.cards, -1)), deck.scan_count))
