def test_configure_mongo_client_for_standalone(self):
self.cleanup()
Script.config = {
'clusterHostInfo': {
'mongos_hosts': [],
'mongodb_hosts': ['node1.test.com'],
'mongodc_hosts': []
}
}
client = MongoClient()
client.configureMongoClient()
self.assertEqual(params.mongos_hosts, 'node1.test.com:27025',
'The mongos_hosts configuration is not right.')
async def main():
connection = await aio_pika.connect_robust("amqp://*****:*****@127.0.0.1/",
loop=asyncio.get_running_loop())
queue_name = "computersQueue"
db_client = MongoClient("houseMonitor", "computers")
async with connection:
# Creating channel
channel = await connection.channel()
# Declaring queue
queue = await channel.declare_queue(queue_name)
with ThreadPoolExecutor(max_workers=MAX_WORKERS) as pool:
async with queue.iterator() as queue_iter:
async for message in queue_iter:
async with message.process():
computer = json.loads(message.body.decode())
asyncio.create_task(
future_wrapper(
asyncio.get_running_loop().run_in_executor(
pool, MongoClient.add_computer, db_client,
computer)))
def run(query='sngularrocks'):
client = MongoClient(host=MONGODB_HOST,
port=MONGODB_PORT,
db=MONGODB_DATABASE)
youtube_api = YoutubeApi(query)
videos = youtube_api.videos()
for video in videos:
video_id = video.get('id').get('videoId')
snippet = video.get('snippet')
date = (video.get('snippet').get('publishedAt'))[:-5]
data = {
'type': 'youtube',
'internal_id': video_id,
'title': snippet.get('title'),
'description': snippet.get('description'),
'link': '{}?v={}'.format('https://www.youtube.com/watch',
video_id),
'user': {
'name':
snippet.get('channelId'),
'screen_name':
snippet.get('channelName'),
'profile_image_url':
snippet.get('thumbnails').get('default').get('url')
},
'created_at': datetime.strptime(date, '%Y-%m-%dT%H:%M:%S')
}
if client.db.feed.find_one({
'type': 'youtube',
'internal_id': video_id
}) is None:
client.db.feed.insert_one(data)
def test_configure_mongo_client_for_sharding_cluster(self):
self.cleanup()
Script.config['clusterHostInfo'] = {
'mongos_hosts': ['node1.test.com', 'node2.test.com'],
'mongodb_hosts': ['node1.test.com'],
'mongodc_hosts': []
}
# Test without_mongos cluster definition
client = MongoClient()
client.configureMongoClient()
self.assertEqual(params.mongos_hosts,
'node1.test.com:27017,node2.test.com:27017',
'The mongos_hosts configuration is not right.')
# Test with mongos cluster definition
params.mongos_cluster_definition = 'node2.test.com,node1.test.com,node2.test.com'
client.configureMongoClient()
self.assertEqual(
params.mongos_hosts,
'node2.test.com:27017,node1.test.com:27017,node2.test.com:27018',
'The mongos_hosts configuration is not right.')
#!/usr/bin/env python
import sys
sys.path.insert(0, "../")
from optparse import OptionParser
import pymongo
from common.utils import getSiteDBCollection, getSiteDB
import settings
print "This script is outdated. do not use it."
sys.exit(1)
from mongo_client import MongoClient
mongo_client = MongoClient(pymongo.Connection(settings.mongodb_host))
parser = OptionParser()
parser.add_option("-r", "--reset_db", dest="reset_db", help="reset all database of this site(use with caution)",
default="no")
parser.add_option("-i", "--site_id", dest="site_id", help="Site ID(required)", default=None)
parser.add_option("-n", "--site_name", dest="site_name", help="Site Name(required)", default=None)
parser.add_option("-c", "--calc_interval", dest="calc_interval", help="How long (in seconds) the server will update the calculation", default="43200")
(options, args) = parser.parse_args()
site_id, site_name = options.site_id, options.site_name
#assert site_id is not None
#assert site_name is not None
from mongo_client import SameGroupRecommendationResultFilter
logging.basicConfig(format="%(asctime)s|%(levelname)s|%(name)s|%(message)s",
level=logging.WARNING,
datefmt="%Y-%m-%d %I:%M:%S")
def getConnection():
if(settings.replica_set):
return pymongo.MongoReplicaSetClient(settings.mongodb_host, replicaSet=settings.replica_set)
else:
return pymongo.Connection(settings.mongodb_host)
mongo_client = MongoClient(getConnection())
mongo_client.reloadApiKey2SiteID()
# jquery serialize() http://api.jquery.com/serialize/
# http://stackoverflow.com/questions/5784400/un-jquery-param-in-server-side-python-gae
# http://www.tsangpo.net/2010/04/24/unserialize-param-in-python.html
# TODO: referer;
# TODO: when to reload site ids.
class LogWriter:
def __init__(self):
self.local_file = open(settings.local_raw_log_file, "a")
from mongo_client import SameGroupRecommendationResultFilter
logging.basicConfig(format="%(asctime)s|%(levelname)s|%(name)s|%(message)s",
level=logging.WARNING,
datefmt="%Y-%m-%d %I:%M:%S")
def getConnection():
if (settings.replica_set):
return pymongo.MongoReplicaSetClient(settings.mongodb_host,
replicaSet=settings.replica_set)
else:
return pymongo.Connection(settings.mongodb_host)
mongo_client = MongoClient(getConnection())
mongo_client.reloadApiKey2SiteID()
# jquery serialize() http://api.jquery.com/serialize/
# http://stackoverflow.com/questions/5784400/un-jquery-param-in-server-side-python-gae
# http://www.tsangpo.net/2010/04/24/unserialize-param-in-python.html
# TODO: referer;
# TODO: when to reload site ids.
class LogWriter:
def __init__(self):
self.local_file = open(settings.local_raw_log_file, "a")
# coding=utf-8
import sys
sys.path.insert(0, "../")
sys.path.insert(0, "../../")
import csv
import pymongo
import settings
from mongo_client import MongoClient
mongo_client = MongoClient(pymongo.Connection(settings.mongodb_host))
site_id = sys.argv[1]
csv_file = sys.argv[2]
f = open(csv_file, "r")
reader = csv.reader(f)
fields = reader.next()
count = 0
for row in csv.DictReader(f, fieldnames=fields):
count += 1
if count % 1000 == 0:
print count
item = {
"item_id": row["id"],
"item_name": row["item_name"],
"price": row["price"],
"image_link": row["image_link"],
def __init__(self, collection):
self.client = MongoClient(host=MONGODB_HOST,
port=MONGODB_PORT,
db=MONGODB_DATABASE)
self.collection = collection
# coding=utf-8
import sys
sys.path.insert(0, "../")
sys.path.insert(0, "../../")
import csv
import pymongo
import settings
from mongo_client import MongoClient
mongo_client = MongoClient(pymongo.Connection(settings.mongodb_host))
site_id = sys.argv[1]
csv_file = sys.argv[2]
f = open(csv_file, "r")
reader = csv.reader(f)
fields = reader.next()
count = 0
for row in csv.DictReader(f, fieldnames=fields):
count += 1
if count % 1000 == 0:
print count
item = {"item_id": row["id"], "item_name": row["item_name"],
"price": row["price"], "image_link": row["image_link"],
"description": row["description"],
"item_link": "goods.php?id=%s" % row["id"]}
mongo_client.updateItem(site_id, item)
#!/usr/bin/env python
import sys
sys.path.insert(0, "../")
from optparse import OptionParser
import pymongo
from common.utils import getSiteDBCollection, getSiteDB
import settings
print "This script is outdated. do not use it."
sys.exit(1)
from mongo_client import MongoClient
mongo_client = MongoClient(pymongo.Connection(settings.mongodb_host))
parser = OptionParser()
parser.add_option("-r",
"--reset_db",
dest="reset_db",
help="reset all database of this site(use with caution)",
default="no")
parser.add_option("-i",
"--site_id",
dest="site_id",
help="Site ID(required)",
default=None)
parser.add_option("-n",
"--site_name",
dest="site_name",