class MongoDB():
def __init__(self, db_uri, db_name, need_auth=False, auth=()):
self.db = MongoClient(db_uri)[db_name]
if need_auth:
self.db.authenticate(*auth)
def insert_one(self, tb_name, data, encrypt_word):
try:
data['_id'] = hashlib.md5(encrypt_word.encode()).hexdigest()[:16]
rs = self.db[tb_name].insert_one(data)
return True
except Exception as e:
if 'duplicate key error index' in str(e):
return False
print('other mistake -->', e)
return None
def delete_one(self, tb_name, key_value_dict):
try:
self.db[tb_name].delete_one(key_value_dict)
return True
except Exception as e:
return False
def get_one_and_pop_one(self, tb_name):
res = self.db[tb_name].find_one()
if self.delete_one(tb_name, {'_id': res['_id']}):
return res
return None
class TwseDailyDataset(object):
def __init__(self, mongo_url='localhost'):
self.db = MongoClient(host=mongo_url)['twse_daily']
self.db.authenticate(name='crawler', password='******')
self.collections = [
self.db[c]
for c in ['BFI82U', 'FMTQIK', 'MI_INDEX', 'MI_MARGN', 'STOCK_DAY']
]
def date_to_datetime(self, obj_or_year, month=None, day=None):
"""將date物件轉換成datetime物件，時間設為台灣時間的下午2點(收盤時間)。
pymongo寫入datetime物件時，會自動轉換成utc timezone。
"""
if month is None:
year, month, day = obj_or_year.year, obj_or_year.month, obj_or_year.day
else:
year = obj_or_year
dt = datetime(year=year, month=month, day=day, hour=14)
return pytz.timezone('Asia/Taipei').localize(dt)
def __getitem__(self, day):
data = dict()
for c in self.collections:
result = c.find_one({'_id': self.date_to_datetime(day)})
if result:
data.update(result)
return data
def gen_features(self, feature_names, only_trading_day=True):
start_day = date(2004, 4, 1)
end_day = date.today(
) if datetime.now().hour > 15 else date.today() - timedelta(days=1)
dates = []
day = start_day
while day <= end_day:
dates.append(day)
day += timedelta(days=1)
masks = np.zeros(len(dates), dtype=bool)
features = np.zeros((len(dates), len(feature_names)), dtype=np.float32)
cond = {n: 1 for n in feature_names}
cond.update({'_id': 0})
for i, day in enumerate(dates):
feat = {}
for c in self.collections:
result = c.find_one({'_id': self.date_to_datetime(day)}, cond)
if result:
feat.update(result)
if feat:
masks[i] = True
for j, name in enumerate(feature_names):
if name in feat:
features[i, j] = feat[name]
dates = np.array(dates)
if only_trading_day:
features = features[masks]
dates = dates[masks]
return dates, features
def get_db():
# db = MongoClient('140.143.94.171', 27017).crawler
# db.authenticate('mongodbcrawler', 'Shantianci56')
db = MongoClient('10.82.244.18', 27018).crawler
db.authenticate('spider', 'spider')
return db
def getOplog(self):
db = MongoClient(self.mongo_host, self.mongo_port)
if self.mongo_user and self.mongo_password:
db.authenticate(self.mongo_user, self.mongo_password)
dbl = db.local
coll = dbl['oplog.rs']
op_first = (coll.find().sort('$natural', 1).limit(1))
while op_first.alive:
op_fst = (op_first.next())['ts'].time
op_last = (coll.find().sort('$natural', -1).limit(1))
while op_last.alive:
op_last_st = op_last[0]['ts']
op_lst = (op_last.next())['ts'].time
status = round(float(op_lst - op_fst), 1)
self.addMetrics('mongodb.oplog', status)
currentTime = timegm(gmtime())
oplog = int(((str(op_last_st).split('('))[1].split(','))[0])
self.addMetrics('mongodb.oplog-sync', (currentTime - oplog))
class MongoDBHelper:
def __init__(self, config):
self.response_code = None
self.response_data = {}
self.config = config
self.db_address = self.config["db_address"]
self.db_address_uri = self.config["db_address_uri"]
self.db_port = int(self.config["db_port"])
self.db_user = self.config["db_user"]
self.db_pwd = self.config["db_pwd"]
self.db_name = self.config["db_name"]
self.connection = None
self.authenticate_mongo_connection()
def mongo_connection(self):
try:
self.connection = MongoClient(self.db_address, self.db_port)
logging.info("MongoDB Connection is Done !!!")
except Exception as e:
logging.error("MongoDB Connection Error: {}".format(str(e)))
raise Exception("MongoDB Connection Error: {}".format(str(e)))
def authenticate_mongo_connection(self):
try:
connection = MongoClient(self.db_address_uri)
self.connection = connection[self.db_name]
self.connection.authenticate(self.db_user, self.db_pwd)
logging.info("MongoDB Connection is Done !!!")
except Exception as e:
logging.error("MongoDB Auth Connection Error: {}".format(str(e)))
raise Exception("MongoDB Auth Connection Error: {}".format(str(e)))
def get_connection(self):
return self.connection
def run_query(self, query):
try:
self.result_list = []
self.result = self.connection.find(query[0], {'_id': False})
for i in self.result:
self.result_list.append(json.dumps(i))
logging.info("MongoDB Run Filter Query is Done")
except Exception as e:
logging.error("MongoDB Run Filter Query Error: {}".format(str(e)))
raise Exception("MongoDB run Filter Query Error: {}".format(
str(e)))
def insert_to_mongo(self, collection, data):
try:
collection = self.connection[str(collection)]
_insert = collection.insert(data)
logging.info("Insert Done")
except exception as e:
logging.error("Insert to MongoDB Error: {}".format(str(e)))
raise Exception("Insert to MongoDB Error: {}".format(str(e)))
def _get_db_connection(self):
parsed = urlsplit(MONGODB_URI)
db_name = parsed.path[1:]
db = MongoClient(MONGODB_URI)[db_name]
if '@' in MONGODB_URI:
user, password = parsed.netloc.split('@')[0].split(':')
db.authenticate(user, password)
return db
class InnerMongo:
def __init__(self):
self.db = MongoClient('10.82.244.18', 27018).crawler
self.db.authenticate('spider', 'spider')
@property
def db(self):
return self.db
def RouteMongoConn():
db = MongoClient("mongodb://*****:*****@10.10.233.135")["store"]
db.authenticate("writer", "miaoji1109")
collection = db.get_collection("session")
return collection
class OuterMongo:
def __init__(self):
self.db = MongoClient('140.143.94.171', 27017).crawler
self.db.authenticate('mongodbcrawler', 'Shantianci56')
@property
def db(self):
return self.db
class MongoProcessor(BaseProcessor):
def __init__(self, _upstream, _enviroinment):
BaseProcessor.__init__(self, _upstream, _enviroinment)
self._collection = None
def run(self):
self.run_process.start()
def _run(self):
self.logr.debug("Mongo Processor Started")
self.run_loop()
def run_loop(self):
while not self._stopped.is_set():
payload = self.next_message()
if not None == payload:
payload['random'] = random.random()
payload['after_gnip'] = True
self.put_in_mongo(payload)
self.logr.debug("Exiting Mongo run loop")
def put_in_mongo(self, obj):
self.logr.debug("Putting in Mongo: " + str(obj))
for rule in obj['gnip']['matching_rules']:
self.client()[rule['tag']].insert(obj)
def client(self):
if not self._client:
host = self.environment.mongo_host
port = int(self.environment.mongo_port)
db = self.environment.mongo_db
self._client = MongoClient(host=host, port=port)[db]
self._client.authenticate(self.environment.mongo_username,self.environment.mongo_password)
return self._client
# def collection(self):
# if not self._collection:
# self._collection = self.environment.mongo_collection
# return self._collection
def stop(self):
self._stopped.set()
def stopped(self):
return self._stopped.is_set() and self.queue.qsize() == 0
def running(self):
self.run_process.is_alive() and not self.stopped()
def next_message(self):
ret_val = None
if self.queue.qsize() > 0:
try:
ret_val = self.queue.get(block=False)
except multiprocessing.queues.Empty:
self.logr.error("Queue was empty when trying to get next message")
return ret_val
class Game3DmPipeline(object):
def __init__(self):
self.db = MongoClient('master').test
self.db.authenticate('jmw', 'jmw123')
self.collection = self.db.game
def process_item(self, item, spider):
self.collection.insert(dict(item))
return item
def clear_db(param):
db = MongoClient(host=db_config['host'],
port=db_config['port'])[db_config['db_name']]
if db_config['username'] is not None and db_config['password'] is not None:
db.authenticate(db_config['name'], db_config['password'])
db.battles.drop()
db.counter.update({"_id": 0}, {"$set": {"battles": 0, "users": 0}})
print("SUCC")
class Game3DmPipeline(object):
def __init__(self):
self.db = MongoClient('192.168.254.131').game_3dm
self.db.authenticate('gameUser','123456')
self.collection = self.db.game
def process_item(self, item, spider):
self.game.insert(dict(item))
return item
def get_db(host=database.host,
user=database.user,
pwd=database.pwd,
dbname=database.dbname,
rs=database.rs):
'''get db'''
db = MongoClient(host=host, replicaset=rs)[dbname]
if user != '':
db.authenticate(user, pwd)
return db
def mongo_start():
fin = open(sys.argv[2], "r")
text = fin.read()
index = text.split()
cnt = 0
db = MongoClient("127.0.0.1", 27017).test
db.authenticate("test", "test")
cl = db.syntax3
bulk = cl.initialize_unordered_bulk_op()
for file in index:
fin1 = open(file, "r")
text1 = fin1.read()
print file
sentences = parse_text_file(file, text1)
for s in sentences:
tree1 = make_whole_tree(s[2], s[1])
tk = s[1]
tk.append(('', ''))
leaf = []
for i, node in enumerate(tree1):
if len(node['c']) == 0:
leaf.append(node)
retJson = {'r1': [], 'r2': [], 'r3': []}
arg = dict({
'tk': tk,
'tree': tree1,
'nodes': leaf,
'strlist1': retJson['r1'],
'strlist2': retJson['r2'],
'strlist3': retJson['r3'],
})
check_find(arg)
bulk.insert({
'sent': s[0],
'res1': retJson['r1'],
'res2': retJson['r2'],
'res3': retJson['r3']
})
cnt += 1
if cnt % 100 == 0:
# break
result = bulk.execute()
pprint(result)
bulk = cl.initialize_unordered_bulk_op()
result = bulk.execute()
pprint(result)
db.logout()
class MongodbClient(object):
def __init__(self, host=MONGODB_HOST, port=MONGODB_PORT):
self.db = MongoClient(host=host, port=port)[MONGODB_DATABASE]
self.db.authenticate(MONGODB_USER, MONGODB_PASSWORD)
# db = client.admin
# # 认证用户密码
# db.authenticate('root', '123456')
def mongodb_insert(self, table, parameter):
"""
:param table: 表名
:param parameter: 插入mongodb的数据，字典类型
:return: 返回该条插入的数据，方便获取ID ["_id"]
"""
x = self.db[table].insert(parameter)
return str(x)
def mongodb_find_one(self, table, parameter={}):
"""
传入字典类型数据，查找符合条件的，单条数据，并返回字典型数据集合,table为表名
"""
return self.db[table].find_one(parameter)
def mongodb_find(self, table, parameter, projection={}):
"""
传入字典类型数据，查找符合条件的所有数据，并返回字典型数据集合,table为表名
"""
return self.db[table].find(parameter, projection)
def mongodb_remove(self, table, parameter):
"""
传入字典型数据，删除符合条件的数据集,table为表名
"""
self.db[table].remove(parameter)
def mongodb_find_limit(self, table='', param={}, projection={}, limit=0):
return self.db[table].find(param, projection).limit(limit)
def mongodb_count(self, table='', param={}):
return self.db[table].find(param).count()
def mongodb_update(self, table, condition_parameter, result_parameter):
"""
传入条件参数、需要修改的参数，都为字典型，自动添加更新日期,table为表名
"""
self.db[table].update_many(
condition_parameter,
{
"$set": result_parameter,
"$currentDate": {"updateTime": True}
}
)
def connect():
mongo_settings = dict(inexcess.read_config('mongo'))
user = mongo_settings['user']
port = mongo_settings['port']
host = mongo_settings['url']
password = mongo_settings['pass']
db = mongo_settings['db']
uri = 'mongodb://'+user+':'+password+'@'+host+':'+port+'/'+db
database = MongoClient(uri)[db]
database.authenticate(user,password)
return database
class Database(object):
'''
Represent a MongoDB database, backed by pymongo's MongoClient
'''
def __init__(self, **kwargs):
'''
Initialise
'''
if 'uri' in kwargs:
self._client = MongoClient(kwargs['uri'])
if 'db' in kwargs:
self._db = self._client[kwargs['db']]
else:
self._db = self._client.get_default_database()
else:
self._client = MongoClient(
kwargs.get('host', 'localhost'),
kwargs.get('port', 27017)
)
self._db = self._client[kwargs.get('db', 'test')]
self.Document = type('Document', (BaseDocument,), {'_db': self._db})
def authenticate(self, *args, **kwargs):
self._client.authenticate(*args, **kwargs)
def drop(self):
self._client.drop_database(self._db.name)
def drop_collection(self, collection):
if isinstance(collection, str):
self._db.drop_collection(collection)
elif issubclass(collection, BaseDocument):
self._db.drop_collection(collection.__collection__)
else:
raise TypeError
def get_collections(self, include_system_collections=False):
return self._db.collection_names(include_system_collections)
@property
def host(self):
return self._client.host
@property
def port(self):
return self._client.port
@property
def name(self):
return self._db.name
def get_db():
url = os.getenv('MONGO_URI', 'mongodb://*****:*****@' in url:
user, password = parsed.netloc.split('@')[0].split(':')
db.authenticate(user, password)
return db
def mongo_auth():
splited = MONGO_URI.split('//')[-1]
host = splited.split('@')[-1].split(':')[0]
if host == 'localhost':
db = MongoClient(MONGO_URI)
return db.db
port = int(splited.split('@')[-1].split(':')[-1].split('/')[0])
base = splited.split('@')[-1].split('/')[-1]
user = splited.split('@')[0].split(':')[0]
passw = splited.split('@')[0].split(':')[1]
connection = MongoClient(host, port, retryWrites=False)
db = connection[base]
db.authenticate(user, passw)
return db
def mongo_start():
fin = open(sys.argv[2], "r")
text = fin.read()
index = text.split()
cnt = 0
db = MongoClient("127.0.0.1", 27017).test
db.authenticate("test", "test")
cl = db.syntax3
bulk = cl.initialize_unordered_bulk_op()
for file in index:
fin1 = open(file, "r")
text1 = fin1.read()
print file
sentences = parse_text_file(file, text1)
for s in sentences:
tree1 = make_whole_tree(s[2], s[1])
tk = s[1]
tk.append(('', ''))
leaf = []
for i, node in enumerate(tree1):
if len(node['c']) == 0:
leaf.append(node)
retJson = {'r1': [], 'r2': [], 'r3': []}
arg = dict({
'tk': tk,
'tree': tree1,
'nodes': leaf,
'strlist1': retJson['r1'],
'strlist2': retJson['r2'],
'strlist3': retJson['r3'],
})
check_find(arg)
bulk.insert({'sent': s[0], 'res1': retJson['r1'], 'res2': retJson['r2'], 'res3': retJson['r3']})
cnt += 1
if cnt % 100 == 0:
# break
result = bulk.execute()
pprint(result)
bulk = cl.initialize_unordered_bulk_op()
result = bulk.execute()
pprint(result)
db.logout()
class Database:
def __init__(self, environment='local'):
# connect to local database
if environment == 'local':
self.db = MongoClient('mongodb://localhost:27017')['newstime-dev']
# connect to production database
elif environment == 'production':
self.db = MongoClient(
'mongodb://ds129428.mlab.com:29428/')['newstime-prd']
self.db.authenticate('karl', 'karl') # username and passw
else:
print(
"Error: database environment must either be local or production!"
)
return
self.articles = self.db.articles
self.timelines = self.db.timelines
def getArticlesFromTimeline(self, timeline):
return self.articles.find({'_id': {'$in': timeline['articles']}})
def getWaitlistedArticles(self):
return self.articles.find({'waitlisted': True})
def insertTimeline(self, timeline):
return self.timelines.insert_one(timeline).inserted_id
def insertArticle(self, article):
return self.articles.insert_one(article).inserted_id
def updateTimeline(self, timeline):
self.timelines.update_one({'_id': timeline['_id']}, {"$set": timeline},
upsert=False)
def updateArticle(self, article):
self.articles.update_one({'_id': article['_id']}, {"$set": article},
upsert=False)
def isArticleInDatabase(self, link):
# find instead of find_one to get a cursor instead of a document, and limit to 1 result to improve performance
return self.articles.find({'link': link}).limit(1).count() > 0
def removeArticle(self, article):
self.articles.remove({'_id': article['_id']})
def cleanCollections(self):
self.articles.remove({})
self.timelines.remove({})
def mongo_auth():
# 'mongodb://*****:*****@ds125525.mlab.com:25525/heroku_xjf41300'
splited = MONGO_URI.split('//')[-1]
host = splited.split('@')[-1].split(':')[0]
if host == 'localhost':
db = MongoClient(MONGO_URI)
return db.db
port = int(splited.split('@')[-1].split(':')[-1].split('/')[0])
base = splited.split('@')[-1].split('/')[-1]
user = splited.split('@')[0].split(':')[0]
passw = splited.split('@')[0].split(':')[1]
connection = MongoClient(host, port, retryWrites=False)
db = connection[base]
db.authenticate(user, passw)
return db
class MongoDBClient(object):
def __init__(self):
self.__mongodb_username = os.environ['MONGODB_USERNAME']
self.__mongodb_password = os.environ['MONGODB_PASSWORD']
self.__mongodb_host = os.environ['MONGODB_HOST']
self.__mongodb_database_name = os.environ['MONGODB_DATABASE_NAME']
self.__mongodb_port = os.environ['MONGODB_PORT']
self.__mongodb_db = MongoClient(
self.__mongodb_host,
int(self.__mongodb_port))[self.__mongodb_database_name]
self.__mongodb_db.authenticate(self.__mongodb_username,
self.__mongodb_password)
def get_mongodb(self):
return self.__mongodb_db
def get_reviews(self):
"""Collect data for all links in self.courses and store it in MongoDB.
Once self.courses in populated with a list of links, this method will
distribute the links among multiple processes to collect the reviews,
users, and course info for each page of reviews.
"""
dbase = MongoClient()['GolfRecs']
with open('../application/secrets.yaml', 'r') as secrets_file:
secrets = yaml.load(secrets_file)['MongoDB']
dbase.authenticate(secrets['user'], secrets['pass'])
if self.courses.size < 1:
raise Exception(
"No links exist for retrieving reviews. Either call " +
"self.get_courses() or set self.courses equal to a list " +
"of courses you wish to process."
)
handler = DataHandler(self.sessions)
courses_lists = array_split(
self.courses,
self.courses.size // (POOL_SIZE * 10)
)
for courses in courses_lists:
while True:
try:
userpages, courses, reviews = handler.\
get_reviews(dbase, courses)
break
except Exception as err: # pylint: disable=W0703
print('Exception Occurred: {}'.format(err))
renew_connection()
while True:
try:
users = handler.get_users(dbase, userpages)
break
except Exception as err: # pylint: disable=W0703
print(
'Exception Occurred in collecting Users: {}'
.format(err)
)
renew_connection()
renew_connection()
data = (users, courses, reviews)
collections = ['Users', 'Courses', 'Reviews']
filters = ['Username', 'GA Id', 'Review Id']
for args in zip(repeat(dbase), collections, data, filters):
handler.write_documents(*args)
def main():
db = MongoClient(host='10.2.11.234', port=27017)['web_news']
db.authenticate('uestc', 'mongoDB')
dbkey = db['wechatkeys']
for i in dbkey.find():
biz = i['biz']
headers = i['headers']
for k in headers.keys():
headers[k] = headers[k][0]
url = i['url']
seq = '?'
ans = ''
for k in url.keys():
ans += seq + k + '=' + url[k][0]
seq = '&'
url = ans
crawl(biz, url, headers)
def main():
# establing connection to db
try:
#connect = MongoClient("10.3.141.1" ,27017)
#connect.sensors.authenticate(mongo_login, mongo_pass)
#connect = MongoClient('mongodb://' + mongo_login + ':' + mongo_pass + '@127.0.0.1:27017')
connect = MongoClient('10.3.141.1:27017')
connect.authenticate(mongo_login, mongo_pass)
print("Connected successfully!!!")
except:
print("Could not connect to MongoDB")
# connecting or switching to the databases
"""
sensorDB = connect.sensors
# creating or switching to demoCollection
global temp_col
global rssi_col
global pos_col
global ecg_col
temp_col = sensorDB.temp
rssi_col = sensorDB.rssi
pos_col = sensorDB.positionning
ecg_col = sensorDB.ecg
"""
client= paho.Client("client-001")
######Bind function to callback
client.on_message=on_message
#####
print("connecting to broker ",broker,port)
client.connect(broker,port)#connect
client.loop_start() #start loop to process received messages
print("subscribing ")
client.subscribe("anchor/#")#subscribe
# We loop waiting for callbacks
while(True):
time.sleep(4)
def predictOnStream(predictionModelsPath, bufferSize=1000, streamSrc=getTwitterStream, streamSrcConfig={}, streamProcessor=TweetProcessor):
'''
All classes of streamProcessor need to implement the function, process(), as the common interface.
'''
predictors = sorted([fName for fName in next(os.walk(predictionModelsPath))[2] if fName.endswith('.pkl') and fName[0] == 'c'])
tfIdfVectorizers = sorted([fName for fName in next(os.walk(predictionModelsPath))[2] if fName.endswith('.pkl') and fName[0] == 'v'])
predictors = [load(open(predictionModelsPath + pName, 'rb')) for pName in predictors]
tfIdfVectorizers = [load(open(predictionModelsPath + vName, 'rb')) for vName in tfIdfVectorizers]
vecAndPredTuple = tuple(vecAndPred for vecAndPred in izip(tfIdfVectorizers, predictors))
db=MongoClient('ds051459.mongolab.com', 51459)['tanav']
db.authenticate('user', 'password')
tweetsColl=db.tweets
with streamSrc(streamSrcConfig) as stream:
print('Stream established.')
idSet = set()
maxIdPrevBatch, maxIdCurrentBatch = 0, 0
processedTweetBuff = []
rawTxtArr = []
for t in stream:
try:
t = json.loads(t)
except ValueError: # Might be caused by the keep-alive new line character described in https://dev.twitter.com/docs/streaming-apis/connecting#Stalls
continue
if streamProcessor.filter(t):
processedT, rawTxt = streamProcessor.process(t)
rawTxtArr.append(rawTxt)
processedTweetBuff.append(processedT)
if len(processedTweetBuff) > bufferSize:
# append semantic scores
resultsProba = map(lambda vecAndPred: vecAndPred[1].predict_proba(vecAndPred[0].transform(array(rawTxtArr))), vecAndPredTuple)
starRankings = (argmax(resultsProba, 2) + 1.0).T.tolist()
probabilities = amax(resultsProba, 2).T.tolist()
for tweet, starR, proba in izip(processedTweetBuff, starRankings, probabilities):
tweet['semantic_scores'] = starR
tweet['semantic_probabilities'] = proba
tweetsColl.insert(processedTweetBuff) # bulk insert into MongoDB
processedTweetBuff[:] = [] # empty the buffer
rawTxtArr[:] = []
streamProcessor.idSet.clear()
streamProcessor.maxIdPrevBatch = streamProcessor.maxIdCurrentBatch
print('Stream terminated')
def get_connection(env=None):
"""
Gets MongoDB connection. If user and password provided - authenticates (logs in)
:param env: dictionary, example:
{'host': 'ds045454.mongolab.com',
'tz_aware': True,
'max_pool_size': 10,
'port': 45454}
:return: MongoClient
"""
# checking for environment variables
mongo_uri = os.getenv("MONGO_URI")
mongo_db = os.getenv("MONGO_DB")
if mongo_uri and mongo_db:
client = MongoClient(mongo_uri)
log.info("MongoDB environment variables found: %s!" % mongo_uri)
return client[mongo_db]
# environment variables not found, looking for details from configuration file
env = env or default_env
_env = env.copy()
dbname = _env.pop('db', None)
# if auth details supplied - getting details
user = password = None
if 'user' in _env:
user = _env.pop('user')
if 'password' in _env:
password = _env.pop('password')
client = MongoClient(**_env)
if dbname:
log.debug('using db={0}'.format(dbname))
client = getattr(client, dbname)
# authenticating
if user and password:
# if fails - throws exception which will be handled in run_stubo.py
client.authenticate(user, password)
log.info("Login to MongoDB successful!")
return client
def get_connection(env=None):
"""
Gets MongoDB connection. If user and password provided - authenticates (logs in)
:param env: dictionary, example:
{'host': 'ds045454.mongolab.com',
'tz_aware': True,
'max_pool_size': 10,
'port': 45454}
:return: MongoClient
"""
# checking for environment variables
mongo_uri = os.getenv("MONGO_URI")
mongo_db = os.getenv("MONGO_DB")
if mongo_uri and mongo_db:
client = MongoClient(mongo_uri)
log.info("MongoDB environment variables found: %s!" % mongo_uri)
return client[mongo_db]
# environment variables not found, looking for details from configuration file
env = env or default_env
_env = env.copy()
dbname = _env.pop("db", None)
# if auth details supplied - getting details
user = password = None
if "user" in _env:
user = _env.pop("user")
if "password" in _env:
password = _env.pop("password")
client = MongoClient(**_env)
if dbname:
log.debug("using db={0}".format(dbname))
client = getattr(client, dbname)
# authenticating
if user and password:
# if fails - throws exception which will be handled in run_stubo.py
client.authenticate(user, password)
log.info("Login to MongoDB successful!")
return client
def getMaintenance(self):
db = MongoClient(self.mongo_host, self.mongo_port)
if self.mongo_user and self.mongo_password:
db.authenticate(self.mongo_user, self.mongo_password)
host_name = socket.gethostname()
fsync_locked = int(db.is_locked)
config = db.admin.command("replSetGetConfig", 1)
for i in range(0, len(config['config']['members'])):
if host_name in config['config']['members'][i]['host']:
priority = config['config']['members'][i]['priority']
hidden = int(config['config']['members'][i]['hidden'])
self.addMetrics('mongodb.fsync-locked', fsync_locked)
self.addMetrics('mongodb.priority', priority)
self.addMetrics('mongodb.hidden', hidden)
class MongoAgent(object):
def __init__(self, host, port, dbname, username, password):
self.db = MongoClient(host, port)[dbname]
if username != "" and password != "":
self.db.authenticate(username, password)
self.calls_col = self.db["calls"]
self.segments_col = self.db["segments"]
self.sentences_col = self.db["sentences"]
def iter_all(self, c, handler):
for doc in c.find():
if not handler(doc):
break
def find_one_by_key_value(self, c, key, value):
return c.find_one({key: value})
def coll(self, name):
return self.db[name]
def load_config(config):
global _db
global _es
if _db==None:
_db=MongoClient(host=config["host"],port=config["port"]).get_database(config["name"])
if config.has_key("user") and config.get("user","")!="":
if not _db.authenticate(config["user"],config["password"]):
raise Exception("Authenticate error ast {0}:{1}".format(config["host"],config["port"]))
if config.has_key("elasticsearch"):
from elasticsearch import Elasticsearch
if _es==None:
_es=Elasticsearch(config["elasticsearch"])
def load_config(config):
global _db
global _es
if _db == None:
_db = MongoClient(host=config["host"],
port=config["port"]).get_database(config["name"])
if config.has_key("user") and config.get("user", "") != "":
if not _db.authenticate(config["user"], config["password"]):
raise Exception("Authenticate error ast {0}:{1}".format(
config["host"], config["port"]))
if config.has_key("elasticsearch"):
from elasticsearch import Elasticsearch
if _es == None:
_es = Elasticsearch(config["elasticsearch"])
class Mongo(Driver):
def _on_init(self):
self._client = MongoClient(self._config.get(
'host', 'localhost:27017'))[str(self._config.get('db', 'env'))]
self._client.authenticate(str(self._config.user),
password=str(self._config.password))
def g(self, k):
try:
result = self._client.env_conf.find_one({'k': k})
if result is not None:
if self._config.rawValue:
return Value(result)
else:
return Value(result.get('v'), result.get('t'))
except BaseException as e:
logger.critical(e, 'Mongo driver')
pass
raise NotExistsError("Key not exists: %s" % k)
class Config(object):
def __init__(self):
self.MAX_LIMIT = 30
self.MIN_LIMIT = 20
logging.basicConfig(
level=logging.DEBUG,
format=
"%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s",
datefmt="%a, %d %b %Y %H:%M:%S")
self.MGO = MongoClient(
host=environ.get("MongoHost"),
port=int(environ.get("MongoPort")))[environ.get("MongoDB")]
self.MGO.authenticate(name=environ.get("MongoUser"),
password=environ.get("MongoPass"))
AccessKeyId = environ.get('AccessKeyId')
AccessKeySecret = environ.get('AccessKeySecret')
OSS_Endpoint = environ.get('OSS_Endpoint')
self.Bucket = oss2.Bucket(oss2.Auth(AccessKeyId, AccessKeySecret),
OSS_Endpoint, "python-crawler")
def Header(self, protocal, host):
return {
"User-Agent": random.choice(USER_AGENTS),
"Origin": protocal + "://" + host,
"Host": host,
"Referer": protocal + "://" + host,
}
def RandomLimit(self):
return random.choice(range(self.MIN_LIMIT, self.MAX_LIMIT))
def Collection(self, filename):
return path.splitext(path.basename(filename))[0][2:]
def get_db(show_detail=False):
if show_detail:
print("Connect to DB: %s/%s" % (','.join(DB_HOST), DB_NAME))
if DB_REPLSET:
print("Replica Set: %s" % DB_REPLSET)
if DB_USER:
print("Login with user: %s" % DB_USER)
else:
print("Login without user.")
if not DB_REPLSET and len(DB_HOST) > 1:
print('Missing Replica set.')
return None
db = MongoClient(host=DB_HOST, replicaset=DB_REPLSET)[DB_NAME]
if DB_USER != '':
if not db.authenticate(DB_USER, DB_PWD):
print('Database auth failed.')
return None
return db
if not MONGO_CLIENT:
MONGO_PORT = int(getattr(settings, "MONGO_PORT", 27017))
MONGO_HOST = getattr(settings, "MONGO_HOST", "localhost")
MONGO_DB_NAME = getattr(settings, "MONGO_DB_NAME", "test")
MONGO_DB_USER = getattr(settings, "MONGO_DB_USER", False)
MONGO_DB_PASSWORD = getattr(settings, "MONGO_DB_PASSWORD", False)
from pymongo import MongoClient
MONGO_CLIENT = MongoClient(host=MONGO_HOST, port=MONGO_PORT)
MONGO_CLIENT = MONGO_CLIENT[MONGO_DB_NAME]
if MONGO_DB_USER and MONGO_DB_PASSWORD:
MONGO_CLIENT.authenticate(MONGO_DB_USER, MONGO_DB_PASSWORD)
MONGO_DB_VERSION = MONGO_CLIENT.connection.server_info()["versionArray"]
if not float(".".join(MONGO_DB_VERSION[:2])) >= 2.2:
raise ImproperlyConfigured(
"""
Your mongodb service doesn't support TTL
http://docs.mongodb.org/manual/tutorial/expire-data/
"""
)
DB_COLLECTION = MONGO_CLIENT[MONGO_SESSIONS_COLLECTION]
MONGO_SESSIONS_INDEXES = DB_COLLECTION.index_information()
def predict(self):
#args = parse_args()
prototxt = self.defining
caffemodel = self.net
anchors = self.anchors
#print('Called with args:')
#print(args)
cfg_from_file(self.cfg)
print('Using config:')
pprint.pprint(cfg)
while not os.path.exists(caffemodel):
print('Waiting for {} to exist...'.format(caffemodel))
time.sleep(10)
caffe.set_mode_gpu()
caffe.set_device(0)
net = caffe.Net(prototxt, caffemodel, caffe.TEST)
net.name = os.path.splitext(os.path.basename(caffemodel))[0]
print '\n\nLoaded network {:s}'.format(caffemodel)
#img_list = ['tr.jpg']
#for img_name in img_list:
# print '~' * 20
# im_file = os.path.join(cfg.ROOT_DIR, 'data', 'demo', img_name)
# print 'Demo for image: `{:s}`'.format(im_file)
# demo(net, im_file, anchors)
# plt.show()
db = MongoClient('10.76.0.143', 27017)['plate']
db.authenticate('plateReader', 'IamReader')
total_num=db.image.count()
# total_num=1407
#bad=open('bad_list.txt','r')
bad=open('id_path.txt','r')
#bad=open('id_path.txt','r')
#print total_num
#idlist=range(total_num)
#print idlist
error1=0
f=open('re_predict_result_new.txt','w')
for idlist in bad:
#for i in bad:
# num=int(i)-1
#print count
#print num
#print type(num)
re_num=re.split(' ',idlist)
#print re_num[0]
num=int(re_num[0])
#print num
temp=db.image.find_one({'_id':int(num)})
#temp=db.image.find_one({'_id',num+1})
if temp:
#temp=db.image.find_one({'_id':num+1})
im_file=temp['path']
img = exif.load_exif_jpg(im_file)
width = 1280
height = 720
multi = img.shape[0]*img.shape[1]*1.0/(width*height)
multi = math.sqrt(multi)
print multi
resized_img = cv2.resize(img,(int(img.shape[1]/multi), int(img.shape[0]/multi)))
resized_img_name = os.path.join('plate_buffer', 'resized_img.jpg')
cv2.imwrite(resized_img_name, resized_img)
re_img = cv2.imread(resized_img_name)
#e_img_size = re_img.shape
print im_file
try:
presicion= demo(net,resized_img_name,anchors)
print presicion
#print 'result'
#print presicion
#print int(presicion[0][2:3])
#print "a"
#print len(temp['points'])
#print "b"
if len(temp['points'])>0:
f.write(str(num)+' ')
#f.write(str(temp['points'][0][0][0:1][0])+' '+str(temp['points'][0][0][1:2][0])+' ')
#f.write(str(temp['points'][0][1][0:1][0])+' '+str(temp['points'][0][1][1:2][0])+' ')
#f.write(str(temp['points'][0][2][0:1][0])+' '+str(temp['points'][0][2][1:2][0])+' ')
#f.write(str(temp['points'][0][3][0:1][0])+' '+str(temp['points'][0][3][1:2][0])+' ')
#print 'flag1'
if len(presicion)!=0:
f.write('*'+' ')
f.write(str(int(presicion[0][2:3]*multi+1))+' '+str(int(presicion[0][1:2]*multi-1))+' ')
f.write(str(int(presicion[0][2:3]*multi+1))+' '+str(int(presicion[0][3:4]*multi+1))+' ')
f.write(str(int(presicion[0][0:1]*multi-1))+' '+str(int(presicion[0][3:4]*multi+1))+' ')
f.write(str(int(presicion[0][0:1]*multi-1))+' '+str(int(presicion[0][1:2]*multi-1)))
#error1=error1+1
else:
f.write('* '+'0 '+'0 '+'0 '+'0 '+'0 '+'0 '+'0 '+'0')
f.write('\n')
# print 'flags2'
except:
print 'we are wrong'
pass
#break
f.close()
from pymongo import MongoClient, ReturnDocument
#f=open('pld.dat','w')
db = MongoClient('10.76.0.143', 27017)['plate']
db.authenticate('plateReader', 'IamReader')
#for post in db.image.find({'_id':'3'})
a=db.image.find_one({'_id':120762})
#f.write(str(a))
print a
b=db.image.find_one({'_id':524})
print b
c=db.image.find_one({'_id':1008})
print c
print db.image.count()
#f.close()
from pymongo import MongoClient
db = MongoClient().get_database("DATA")
db.authenticate("twitterApplication","gdotwitter", source="admin")
print db.get_collection("Brexit_old").find({},{"retweetedStatus":1}).limit(1)[0]
elif c == '(':
ntr += tmp + '('
state[0] = 1
tmp = ''
else:
tmp += c
return ntr
if __name__ == '__main__':
fin = open("index.txt", "r")
addr = '127.0.0.1'
db = MongoClient(addr).test
db.authenticate('test', 'test')
posts = db.syntax2
text = fin.read()
index = text.split()
info = []
cnt = 0
for file in index:
fin1 = open(file, "r")
text1 = fin1.read()
print file
sentences = parse_text_file(file, text1)
cnt += 1
for s in sentences:
tr = make_tree(s[2])
ntr = make_tree0(tr)
from flask import Flask, render_template, jsonify, request
from random import randint
from pymongo import MongoClient
from bson.objectid import ObjectId
app = Flask(__name__)
app.Debug = True
db = MongoClient('mongodb://*****:*****@troup.mongohq.com:10079/piloto-tweets')['piloto-tweets']
db.authenticate('yamil', 'medalla')
tweets = db.tweets
@app.route('/', methods = ['GET'])
def view_tweets():
return render_template('index.html', tweets = list(tweets.find()))
@app.route('/tweets', methods = ['GET'])
def return_all_tweets():
return jsonify({"tweets": list(tweets.find())})
@app.route('/tweets', methods = ['POST'])
def create_tweet():
# tweets.append({"id": randint(0,256), "text": request.form['tweet']})
return jsonify({'_id': str(tweets.insert({'text': request.form['tweet']}))})
@app.route('/tweets/<string:id>', methods = ['GET', 'DELETE'])
def handle_single_tweet(id): #bregar con un solo tweet para ver si lo borra o lo muestra
if request.method == 'DELETE':
return jsonify({tweets.remove({'_id': ObjectId(id)})})
else:
return jsonify(tweets.find_one({'_id': ObjectId(id)}))
from pymongo import MongoClient as Connection from config import DB_HOST, MONGO_AUTHENTICATE MongoDB= Connection(DB_HOST, wtimeout=1).dragon MongoDB.authenticate(MONGO_AUTHENTICATE['id'], MONGO_AUTHENTICATE['passwd'])
from pprint import pprint
from pymongo import MongoClient
from bson import ObjectId
dbuser = "******"
dbpasswd = "monitor678"
remote = MongoClient("nuev9.com")['monitor']
remote.authenticate(dbuser, dbpasswd)
local = MongoClient()['monitor']
"""
laccs = set([x['_id'] for x in local.accounts.find()])
accs = remote.accounts.find({"_id": ObjectId("54f4b34941b84c23d147babd")})
#pprint(accs[0])
for acc in accs:
#if acc['_id'] not in laccs:
local.accounts.save(acc)
"""
acc = remote.accounts.find_one({"_id": ObjectId("54f4b34941b84c23d147babd")})
#pprint (acc)
#acc['campaigns'] = {"iae1": acc['campaigns']['54f4b32141b84c23d147babc']}
#acc['name'] = "INSIGHT-DRINKS-IAE-GRUPO-1"
#acc['users'] = {'iae1': {u'access': u'admin', u'password': u'', u'username': u'iae1'}}
campaign = acc["campaigns"]["54f4b32141b84c23d147babc"]
for x in range(9,10):
newacc = acc
newacc["_id"] = ObjectId()
newacc["campaigns"] = {"iae%s"%x: campaign}
newacc["name"] = "INSIGHT-DRINKS-IAE-GRUPO-%s" % x
def predict(self):
#args = parse_args()
prototxt = self.defining
caffemodel = self.net
anchors = self.anchors
#print('Called with args:')
#print(args)
cfg_from_file(self.cfg)
print('Using config:')
pprint.pprint(cfg)
while not os.path.exists(caffemodel):
print('Waiting for {} to exist...'.format(caffemodel))
time.sleep(10)
caffe.set_mode_gpu()
caffe.set_device(0)
net = caffe.Net(prototxt, caffemodel, caffe.TEST)
net.name = os.path.splitext(os.path.basename(caffemodel))[0]
print '\n\nLoaded network {:s}'.format(caffemodel)
#img_list = ['tr.jpg']
#for img_name in img_list:
# print '~' * 20
# im_file = os.path.join(cfg.ROOT_DIR, 'data', 'demo', img_name)
# print 'Demo for image: `{:s}`'.format(im_file)
# demo(net, im_file, anchors)
# plt.show()
db = MongoClient('10.76.0.143', 27017)['plate']
db.authenticate('plateReader', 'IamReader')
total_num=db.image.count()
total_num=103470
#bad=open('bad_list.txt','r')
#idlist=range(total_num)
#print idlist
error1=0
f=open('predict_result1.txt','w')
for num in range(total_num):
#for i in bad:
# num=int(i)-1
#print count
temp=db.image.find_one({'_id':num+1})
#temp=db.image.find_one({'_id',num+1})
if temp:
#temp=db.image.find_one({'_id':num+1})
im_file=temp['path']
print im_file
try:
presicion= demo(net,im_file,anchors)
print presicion
#print 'result'
#print presicion
#print int(presicion[0][2:3])
#print temp['points']
if len(temp['points'])>0:
f.write(str(num+1)+' ')
f.write(str(temp['points'][0][0][0:1][0])+' '+str(temp['points'][0][0][1:2][0])+' ')
f.write(str(temp['points'][0][1][0:1][0])+' '+str(temp['points'][0][1][1:2][0])+' ')
f.write(str(temp['points'][0][2][0:1][0])+' '+str(temp['points'][0][2][1:2][0])+' ')
f.write(str(temp['points'][0][3][0:1][0])+' '+str(temp['points'][0][3][1:2][0])+' ')
if len(presicion)!=0:
f.write('*'+' ')
f.write(str(int(presicion[0][2:3]))+' '+str(int(presicion[0][1:2]))+' ')
f.write(str(int(presicion[0][2:3]))+' '+str(int(presicion[0][3:4]))+' ')
f.write(str(int(presicion[0][0:1]))+' '+str(int(presicion[0][3:4]))+' ')
f.write(str(int(presicion[0][0:1]))+' '+str(int(presicion[0][1:2])))
#error1=error1+1
else:
f.write('* '+'0 '+'0 '+'0 '+'0 '+'0 '+'0 '+'0 '+'0')
f.write('\n')
except:
print 'we are wrong'
pass
class DotaDB:
def __init__(self):
self.db = MongoClient().dota2
self.db.authenticate("<REPLACE WITH YOUR MONGO USER>","<REPLACE WITH YOUR MONGO PASSWORD>")
self.matches = self.db.matches
self.heroes = self.db.heroes
self.players = self.db.players
self.dotaApi = DotaApi()
def vanityTo64(self,vanityName):
"""returns the 64 bit steam value thing"""
player = self.players.find_one({"name":vanityName})
if not player:
player = self.createPlayer(vanityName)
return int(player["steam64"])
def vanityTo32(self,vanityName):
"""returns the 32 bit steam value thing"""
player = self.players.find_one({"name":vanityName})
if not player:
player = self.createPlayer(vanityName)
return int(player["steam32"])
def steam32ToVanity(self,steam32,useApi=True):
""" returns the vanity name of someone """
if steam32 == 4294967295:
return "Unknown"
name = self.players.find_one({"steam32": steam32})
if name:
return name["name"]
if not useApi:
return False
apiName = self.dotaApi.getPlayerBySteam32(steam32)
if apiName:
return apiName["response"]["players"][0]["personaname"]
else:
return False
def heroIdtoName(self,id):
"""fetches heroes from the database"""
heroes = self.heroes.find_one()
if not heroes or int(time.time()) - heroes["lastUpdate"] > 86400:
res = self.dotaApi.getheroes()
if not res.has_key("error"):
self.heroes.remove()
self.heroes.save({"lastUpdate": int(time.time()), "data": res})
heroes = self.heroes.find_one()
for heroe in heroes["data"]["result"]["heroes"]:
if heroe["id"] == id:
return heroe["localized_name"]
def createPlayer(self,vanityName,steam64=None):
"""creates the user in the database or updates it for watever reason"""
if steam64:
apiPlayer = {"64bits":steam64,"32bits":int(steam64)-76561197960265728}
else:
apiPlayer = self.dotaApi.getPlayerByName(vanityName)
self.db.players.remove({"name":vanityName.lower()})
self.db.players.save({"name":vanityName,"lastUpdated":int(time.time()),"steam64":apiPlayer["64bits"],"steam32":apiPlayer["32bits"]})
return self.db.players.find_one({"name":vanityName})
def getMatch(self,matchId):
"""returns the match from the database OR fetches it from the API and stores it on the database"""
match = self.matches.find_one({"match_id":matchId})
if not match:
apiMatch = self.dotaApi.getMatch(matchId)
if apiMatch.has_key("error"):
return None
else:
self.matches.save({"match_id":matchId,"match_data": apiMatch})
match = self.matches.find_one({"match_id":matchId})
return match["match_data"]["result"]
def getMatches(self,vanityName):
""" returns an array of matches given the steam vanity name """
player = self.players.find_one({"name":vanityName})
if player and player.has_key("matches") and int(time.time()) - player["matches"]["lastUpdate"] < 3600:
return player["matches"]["data"]
else:
matches = self.dotaApi.getmatches(self.vanityTo32(vanityName),None,5)
if type(matches) is dict and matches.has_key("error"):
return None
else:
self.players.update({"name":vanityName},{"$set": {"matches": { "data": matches["matches"],"lastUpdate": int(time.time())}}})
return self.players.find_one({"name":vanityName})["matches"]["data"]
def getFullMatchesList(self,vanityName,download=True):
""" generates a list in the database of matches """
needMore = True
startingMatch = None
list = self.players.find_one({"name":vanityName})
if list and list.has_key("fullMatchesList"):
list = list["fullMatchesList"]
else:
list = []
if not download:
return list
while(needMore):
apiRes = self.dotaApi.getmatches(self.vanityTo32(vanityName),startingMatch,25)
if apiRes:
if apiRes["results_remaining"] == 0:
needMore = False
apiList = [x["match_id"] for x in apiRes["matches"]]
newList = []
for i in apiList:
if list.count(i) == 0:
newList.append(i)
else:
needMore = False
startingMatch = i
list = newList + list
else:
return False
self.players.update({"name":vanityName},{"$set" : {"fullMatchesList": list }})
return list
def matchGamesList(self,list):
""" returns the full games list"""
return self.matches.find({"match_id": {"$in": list}})
def wipeGames(self,list):
""" removes the games from this player from the database"""
return self.matches.remove({"match_id": {"$in": list}})
def delUser(self,vanityName):
""" deletes an user from the database"""
self.players.remove({"name":vanityName})
return True
class Crawler(Spider):
name = "2"
allowed_domains = ["scholar.google.com"]
def __init__(self):
self.db = MongoClient("166.111.7.105", 30017)["bigsci"]
self.db.authenticate("", "")
conf = open("conf").readlines()
profile_db = conf[1].strip()
limit = int(conf[0].strip())
self.profile = self.db[profile_db]
self.start_urls = []
self._id = {}
self.count = 0
t = datetime.now()
tmp = {
"year": t.year,
"month": t.month,
"day": t.day,
"hour": t.hour,
"minute": t.minute,
"second": t.second,
"microsecond": t.microsecond,
}
self.fr = []
for i in self.profile.find({}, {"_id": 1, "ID": 1, "url": 1, "token": 1}):
self.fr.append(i)
random.shuffle(self.fr)
self.length = len(self.fr)
print "Total author count: %d" % self.length
self.cnt = 0
while self.cnt < self.length:
self.entry = self.fr[self.cnt]
if not self.isVaild(tmp):
self._id[self.entry["ID"]] = self.entry["_id"]
url = self.entry["url"] + "&cstart=0&pagesize=100"
self.start_urls.append(url)
self.cnt += 1
print "total url count: %d" % len(self.start_urls)
self.proxies = []
self.request20proxy = "http://erwx.daili666.com/ip/?tid=558045424788230&num=20&foreign=only"
self.request1proxy = "http://erwx.daili666.com/ip/?tid=558045424788230&num=1&foreign=only"
proxy = urllib.urlopen(self.request20proxy)
for line in proxy.readlines():
self.proxies.append("http://" + line.strip())
# self.limit = len(self.idList)
# self.entry = self.profile.find_one({"_id":self.idList[self.ptr]})
# while self.isVaild(tmp):
# print "Skip %s" % str(self.entry["_id"])
# self.ptr += 1
# if self.ptr >= self.limit:
# self.entry = None
# break
# self.entry = self.profile.find_one({"_id":self.idList[self.ptr]})
# if self.entry is not None:
# self.entry['token'] = tmp
# self.entry['pubs'] = []
# self.start_urls = [self.entry['url'] + '&cstart=0&pagesize=100',]
# print "Start Processing %s" % str(self.entry["_id"])
def choose_proxy(self):
idx = random.randint(0, 19)
if not self.test_proxy(self.proxies[idx]):
proxy = urllib.urlopen(self.request1proxy)
self.proxies[idx] = "http://" + proxy.readlines()[0].strip()
print "Proxy " + self.proxies[idx] + " is added."
return self.proxies[idx]
def test_proxy(self, proxy):
socket.setdefaulttimeout(3.0)
test_url = "http://scholar.google.com"
try:
f = urllib.urlopen(test_url, proxies={"http": ":@" + proxy})
except:
print "Proxy " + proxy + " fails!"
return False
else:
if f.getcode() != "200":
print "Proxy " + proxy + " fails!"
return False
else:
return True
def make_requests_from_url(self, url):
request = Request(url, callback=self.parse)
request.meta["proxy"] = self.choose_proxy()
request.headers["Proxy-Authorization"] = ""
return request
def parse(self, response):
sel = Selector(response)
url = response.url
idx = url.find("user")
_id = url[idx + 5 : idx + 17]
item = CrawlerItem()
item["url"] = url
item["_id"] = self._id[_id]
# item['ID'] = _id
# item['url'] = response.url
# item['name'] = sel.xpath('//div[@id="gsc_prf_in"]/text()').extract()[0]
# item['info'] = sel.xpath('//div[@class="gsc_prf_il"]/text()').extract()[0]
t = datetime.now()
item["token"] = {
"year": t.year,
"month": t.month,
"day": t.day,
"hour": t.hour,
"minute": t.minute,
"second": t.second,
"microsecond": t.microsecond,
}
tmp = sel.xpath('//tbody[@id="gsc_a_b"]/tr[@class="gsc_a_tr"]/td[@class="gsc_a_t"]/a/text()').extract()
item["pubs"] = []
n = len(tmp)
for i in range(1, n + 1):
pub = {}
pub["title"] = sel.xpath(
'//tbody[@id="gsc_a_b"]/tr[@class="gsc_a_tr"][%d]/td[@class="gsc_a_t"]/a/text()' % i
).extract()
pub["url"] = sel.xpath(
'//tbody[@id="gsc_a_b"]/tr[@class="gsc_a_tr"][%d]/td[@class="gsc_a_t"]/a/@href' % i
).extract()
pub["author"] = sel.xpath(
'//tbody[@id="gsc_a_b"]/tr[@class="gsc_a_tr"][%d]/td[@class="gsc_a_t"]/div[1]/text()' % i
).extract()
pub["venue"] = sel.xpath(
'//tbody[@id="gsc_a_b"]/tr[@class="gsc_a_tr"][%d]/td[@class="gsc_a_t"]/div[2]/text()' % i
).extract()
pub["citation"] = sel.xpath(
'//tbody[@id="gsc_a_b"]/tr[@class="gsc_a_tr"][%d]/td[@class="gsc_a_c"]/a/text()' % i
).extract()
pub["year"] = sel.xpath(
'//tbody[@id="gsc_a_b"]/tr[@class="gsc_a_tr"][%d]/td[@class="gsc_a_y"]/span/text()' % i
).extract()
item["pubs"].append(pub)
self.count += 1
print "current count: %d,total count: %d" % (self.count, self.length)
yield item
if n == 100:
offset = 0
d = 0
idx = url.find("cstart=")
idx += 7
while url[idx].isdigit():
offset = offset * 10 + int(url[idx])
idx += 1
d += 1
request = Request(url[: idx - d] + str(offset + 100) + "&pagesize=100", callback=self.parse)
request.meta["proxy"] = self.choose_proxy()
request.headers["Proxy-Authorization"] = ""
yield request
# else:
# yield item
# self.ptr += 1
# if self.ptr >= self.limit:
# self.entry = None
# else:
# self.entry = self.profile.find_one({"_id":self.idList[self.ptr]})
# while self.isVaild(item['token']):
# print "Skip %s" % str(self.entry["_id"])
# self.ptr += 1
# if self.ptr >= self.limit:
# self.entry = None
# break
# self.entry = self.profile.find_one({"_id":self.idList[self.ptr]})
# if self.entry is not None:
# self.entry['token'] = item['token']
# self.entry['pubs'] = []
# yield Request(self.entry['url'] + '&cstart=0&pagesize=100', callback = self.parse, meta={'proxy'='http://localhost:3128'})
# print "Start Processing %s" % str(self.entry["_id"])
def isVaild(self, new):
if self.entry is None:
print "Invaild item"
return False
if not self.entry.has_key("token"):
print "No token"
return False
t = self.entry["token"]
old = datetime(t["year"], t["month"], t["day"], t["hour"], t["minute"], t["second"], t["microsecond"])
t = new
new = datetime(t["year"], t["month"], t["day"], t["hour"], t["minute"], t["second"], t["microsecond"])
d = new - old
print "Have passed %d days after last update" % d.days
return d.days <= 50
import sys
from pymongo import MongoClient
from Crypto.Hash import SHA256
#####################################
# main program #
#####################################
mongoURL = sys.argv[1]
mongoEAEPassword = sys.argv[2]
adminPwd = sys.argv[3]
hash = SHA256.new()
hash.update("eae")
db = MongoClient('mongodb://' + mongoURL + '/').eae
db.authenticate('eae', mongoEAEPassword, mechanism='SCRAM-SHA-1')
adminUser = {"username": "******",
"password": hash.hexdigest()
}
db.users.insert_one(adminUser)
# -*- coding: utf-8 -*-
from pymongo import MongoClient
db = MongoClient("localhost", 27017).billionaire
name = raw_input("name: ")
pw = raw_input("pw: ")
db.authenticate(name, pw)
print db.gift.ensure_index("uid")
print db.sponsor.ensure_index("uid")
class setting:
def __init__(self,cfg):
self.logined = False
self.cfg = cfg
self.client = MongoClient(cfg['db_ip'],cfg['db_port']).bot
self.client.authenticate(cfg['db_user'],cfg['db_pass'])
self.logined = False
def login(self,user,password,id):
if self.logined:
return True
userdatas = self.client.user.find({"name":user})
if userdatas.count() == 0:
return False
else:
currect = userdatas[0]['pass']
import hashlib
m = hashlib.md5()
m.update(currect + str(id))
currect = m.hexdigest()
if currect == password:
self.user = user
self.logined = True
return True
else:
return False
def getdata(self):
if self.logined:
userdata = self.client.user.find({"name":self.user})[0]
return userdata['data']
else:
return None
def cleck_user(self,name):
userdatas = self.client.user.find({"name":name})
if userdatas.count() == 0:
return False
else:
return True
def register(self,name):
self.client.user.insert_one({
'name': name
})
def change_nick(self,who,twitchid,nick):
try:
nick_userdata = self.client.user.find({"name": who})[0]['data']['spe_name']
except:
self.client.user.update_one({"name": who},{"$set":{"data.spe_name":[]}})
nick_userdata = self.client.user.find({"name": who})[0]['data']['spe_name']
exist = False
for nickdata in nick_userdata:
if nickdata['name'] == twitchid:
nickdata['nick'] = nick
exist = True
break
if not exist:
nick_userdata.append({
'name': twitchid,
'nick': nick
})
self.client.user.update_one({"name": who},{'$set':{'data.spe_name':nick_userdata}})
return exist
def rm_nick(self,who,twitch):
try:
nick_userdata = self.client.user.find({"name": who})[0]['data']['spe_name']
except:
self.client.user.update_one({"name": who},{"$set":{"data.spe_name":[]}})
nick_userdata = self.client.user.find({"name": who})[0]['data']['spe_name']
exist = False
for i in xrange(len(nick_userdata)):
if nick_userdata[i]['name'] == twitch:
nick_userdata.pop(i)
exist = True
break
self.client.user.update_one({"name": who},{'$set':{'data.spe_name':nick_userdata}})
return exist
def nick(self,room,id):
try:
nick_userdata = self.client.user.find({"name": room})[0]['data']['spe_name']
for nickdata in nick_userdata:
if nickdata['name'] == id:
return nickdata['nick']
break
return id
except:
return id
def token_check(self,user,token):
userdata = self.client.user.find({"name": user,"room_token": token})
if userdata.count() == 0:
return False
else:
return True
def get_room_token(self,user):
userdata = self.client.user.find({"name": user})
if userdata[0].has_key('room_token'):
return userdata[0]['room_token']
else:
token = self.rd(50)
self.client.user.update_one({"name": user},{'$set':{'room_token':token}})
return token
def rd(self,N):
import random,string
return ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(N))
def canEditNickInRoom(self,user):
userdata = self.client.user.find({"name": user})
if userdata[0].has_key('room_edit_nick'):
return userdata[0]['room_edit_nick']
else:
self.client.user.update_one({"name": user},{'$set':{'room_edit_nick':False}})
return False
def setEditNickInRoom(self,user,value):
self.client.user.update_one({"name": user},{'$set':{'room_edit_nick':value}})
def cando(self,user,dowhat):
userdata = self.client.user.find({"name": user})
if userdata[0].has_key(dowhat):
return userdata[0][dowhat]
else:
self.client.user.update_one({"name": user},{'$set':{dowhat:False}})
return False
def setCanDo(self,user,dowhat,value):
self.client.user.update_one({"name": user},{'$set':{dowhat:value}})
