def main(start_from=None):
# Load config information
load_config()
# Connect to RabbitMQ Queue
connection, channel = get_queue_channel(RABBIT_HOST)
# Connect to databse
client = MongoClient(MONGO_HOST, MONGO_PORT)
db = client[MONGO_DB]
db.authenticate(MONGO_USER, MONGO_PWD)
db_repos = db[MONGO_COLL]
last_id = start_from # Last id variable to be advanced
reauth = True # Reauth variable to check if we need to reauthenticate for GitHub
gh = None # Just for good measure
while 1:
# Authenticate on GitHub and get all repos
if reauth:
gh = github3.login(GH_USERS[GH_CUR_USR]['login'], GH_USERS[GH_CUR_USR]['pwd'])
repos = gh.iter_all_repos(since=last_id)
# Crawl repos
reauth, last_id = start_crawl(repos, db_repos, gh, channel, last_id)
#Close connection to databse
client.close()
#Close connection to queue
channel.close()
connection.close()
def yearStatistics(request):
#Código de mongodb
client = MongoClient(ip, port)
db = client['twitterdata']
tweetCollection = db['tweets']
#Tweets por meses
result = tweetCollection.map_reduce(mapCuentaMeses, reduce,"myresult6")
for doc in result.find():
result.update({'_id':doc['_id']}, {'$set': {'date':doc['_id']}})
anio = result.find(limit=10).sort('date',1)
anioArray = []
for a in anio:
anioArray.append(a['value'])
anio = result.find(limit=10).sort('date',1)
# Retweets
result2 = tweetCollection.map_reduce(mapCuentaMesesRT, reduce,"myresult6")
for doc in result2.find():
result2.update({'_id':doc['_id']}, {'$set': {'date':doc['_id']}})
anioRT = result2.find(limit=10).sort('date',1)
anioArrayRT = []
for a in anioRT:
anioArrayRT.append(a['value'])
anioRT = result2.find(limit=10).sort('date',1)
client.close()
#return the template
return render_to_response('yearStatistics.html', locals())
class MongoIterator(object):
def __init__(self, uri, db, collection, skip=0, limit=0, filter=None):
self._collection = MongoClient(uri)[db][collection]
self._skip = skip
self._limit = limit
self._filter = filter
def __iter__(self):
return self.stream()
def stream(self, conditions=None, projection=None, skip=None, limit=None):
proj = {k: 1 for k in projection} if projection else {}
if proj:
proj.update({'_id': False}) # skip internal id
return self._collection.find(conditions or self._filter, proj or None, skip=skip or self._skip, limit=limit or self._limit)
def size(self):
return self._collection.count() if not self._filter else self._collection.find(self._filter).count()
@property
def filter(self):
return self._filter
@filter.setter
def filter(self, conditions):
self._filter = conditions
class MongoStore(Store):
def __init__(self, subscription_id, config):
super().__init__(subscription_id)
self._client = MongoClient(config['MONGO_HOST'],
config['MONGO_PORT'])
self._db = self._client.get_database(config['MONGO_DATABASE'])
self._collection = self._db[config['MONGO_COLLECTION']]
def set_state(self, state):
self._collection.replace_one({'_id': self._subscription_id}, state, upsert=True)
def set_value(self, key, value):
self._collection.update_one({'_id': self._subscription_id}, {'$set': {key: value}}, upsert=True)
def push_all(self, key, values):
self._collection.update_one({'_id': self._subscription_id}, {'$push': {key: {'$each': values}}}, upsert=True)
def get_value(self, key, default=None):
state = self.get_state()
if state:
return state.get(key, default)
return default
def get_state(self):
return self._collection.find_one({'_id': self._subscription_id})
def get_collection(self):
return self._collection
def close(self):
self._client.close()
def run(host,database,graphname):
# Create an empty response object.
response = {}
collectionNames = []
# this method traverses the documents in the selected graph collection and builds a JSON object
# that represents the graph to the application. It might be faster to adopt to using a standard
# networkX JSON description, but this is certainly simple and flexible for an initial prototype.
client = MongoClient(host, 27017)
db = client[database]
# get a list of all collections (excluding system collections)
collection = db[graphname]
# loop through the records in the network and take the appropriate action for each type
nodecount = collection.find({'type':'node'}).count()
edgecount = collection.find({'type':'link'}).count()
# Pack the results into the response object, and return it.
response['result'] = {}
response['result']['nodes'] = nodecount
response['result']['links'] = edgecount
client.close()
# Return the response object.
#tangelo.log(str(response))
return json.dumps(response)
def test_cert_ssl_implicitly_set(self):
# Expects the server to be running with the server.pem, ca.pem
# and crl.pem provided in mongodb and the server tests eg:
#
# --sslPEMKeyFile=/path/to/pymongo/test/certificates/server.pem
# --sslCAFile=/path/to/pymongo/test/certificates/ca.pem
# --sslCRLFile=/path/to/pymongo/test/certificates/crl.pem
#
# Also requires an /etc/hosts entry where "server" is resolvable
if not CERT_SSL:
raise SkipTest("No mongod available over SSL with certs")
client = ssl_client
response = ssl_client.admin.command('ismaster')
if 'setName' in response:
client = MongoClient(pair,
replicaSet=response['setName'],
w=len(response['hosts']),
ssl_certfile=CLIENT_PEM)
db = client.pymongo_ssl_test
db.test.drop()
db.test.insert_one({'ssl': True})
self.assertTrue(db.test.find_one()['ssl'])
client.drop_database('pymongo_ssl_test')
def test_cert_ssl_uri_support(self):
# Expects the server to be running with the server.pem, ca.pem
# and crl.pem provided in mongodb and the server tests eg:
#
# --sslPEMKeyFile=/path/to/pymongo/test/certificates/server.pem
# --sslCAFile=/path/to/pymongo/test/certificates/ca.pem
# --sslCRLFile=/path/to/pymongo/test/certificates/crl.pem
#
# Also requires an /etc/hosts entry where "server" is resolvable
if not CERT_SSL:
raise SkipTest("No mongod available over SSL with certs")
if not SERVER_IS_RESOLVABLE:
raise SkipTest("No hosts entry for 'server'. Cannot validate "
"hostname in the certificate")
uri_fmt = ("mongodb://server/?ssl=true&ssl_certfile=%s&ssl_cert_reqs"
"=%s&ssl_ca_certs=%s")
client = MongoClient(uri_fmt % (CLIENT_PEM, 'CERT_REQUIRED', CA_PEM))
db = client.pymongo_ssl_test
db.test.drop()
db.test.insert_one({'ssl': True})
self.assertTrue(db.test.find_one()['ssl'])
client.drop_database('pymongo_ssl_test')
def __init__(self):
self.__cfgReporting = Config(os.path.join(RAGPICKER_ROOT, "config", "reporting.conf"))
self.__cfgProcessing = Config(os.path.join(RAGPICKER_ROOT, "config", "processing.conf"))
self.__mongodbEnabled = self.__cfgReporting.getOption("mongodb", "enabled")
self.__codedbEnabled = self.__cfgReporting.getOption("codeDB", "enabled")
self.__bluecoatEnabled = self.__cfgProcessing.getOption("all_bluecoatMalwareAnalysisAppliance", "enabled")
if self.__mongodbEnabled:
# Anbindung an Datenbank MongoDB Collection Ragpicker herstellen
try:
mongodbHost = self.__cfgReporting.getOption("mongodb", "host")
mongodbPort = self.__cfgReporting.getOption("mongodb", "port")
self.__mongodbConnection = MongoClient(mongodbHost, mongodbPort)
self.__mongodbCollectionRagpicker = self.__mongodbConnection.MalwareAnalyse.ragpicker
self.__mongodbCollectionFamilies = self.__mongodbConnection.MalwareAnalyse.families
self.__mongodbCollectionSandboxTaskQueue = self.__mongodbConnection.MalwareAnalyse.sandboxTaskQueue
except TypeError:
raise Exception("MongoDB connection port in report.config must be integer")
except ConnectionFailure:
raise Exception("Cannot connect to MongoDB (ragpicker)")
if self.__codedbEnabled:
# Anbindung an Datenbank MongoDB Collection CodeDB herstellen
try:
codedbHost = self.__cfgReporting.getOption("codeDB", "mongo_db_host")
codedbPort = self.__cfgReporting.getOption("codeDB", "mongo_db_port")
self.__codedbConnection = MongoClient(codedbHost, codedbPort)
self.__codedbCollectionCodedb = self.__codedbConnection.MalwareAnalyse.codeDB
except TypeError:
raise Exception("MongoDB connection port for CodeDB in report.config must be integer")
except ConnectionFailure:
raise Exception("Cannot connect to MongoDB (codeDB)")
def load_20_news_group():
""" Loads the 20 news group corpus into
a mongo database
"""
mc = MongoClient()
db = mc["astrology"]
coll_name = "corpora.twenty_news_group"
meta_coll_name = "corpora.twenty_news_group.meta"
# Drop if already exists
db.drop_collection(coll_name)
db.drop_collection(meta_coll_name)
coll = db[coll_name]
meta_coll = db[meta_coll_name]
labels = set()
for batch in get_20_news_group(300, labels):
coll.insert_many(batch)
meta_doc = {"labels": list(labels)}
meta_coll.insert_one(meta_doc)
coll.create_index("label")
mc.close()
def lookup_phenotype_results_by_id(id_list: list):
client = MongoClient(util.mongo_host, util.mongo_port)
db = client[util.mongo_db]
obj = dict()
obj['results'] = list()
obj['indexes'] = dict()
try:
# db.phenotype_results.find({"_id": { $in: [ObjectId("5b117352bcf26f020e392a9c"), ObjectId("5b117352bcf26f020e3926e2")]}})
# TODO TODO TODO
ids = list(map(lambda x: ObjectId(x), id_list))
res = db.phenotype_results.find({
"_id": {
"$in": ids
}
})
obj['results'] = list(res)
n = 0
for o in obj['results']:
id = str(o['_id'])
obj['indexes'][id] = n
n = n + 1
except Exception as e:
traceback.print_exc(file=sys.stdout)
obj['success'] = False
finally:
client.close()
return obj
def phenotype_subjects(job_id: str, phenotype_final: bool):
client = MongoClient(util.mongo_host, util.mongo_port)
db = client[util.mongo_db]
results = []
# db.phenotype_results.aggregate([ {"$match":{"job_id":{"$eq":10201}, "phenotype_final":{"$eq":true}}},
# {"$group" : {_id:"$subject", count:{$sum:1}}} ])
try:
q = [
{
"$match": {
"phenotype_final": {
"$eq": phenotype_final
},
"job_id": {
"$eq": int(job_id)
}
}},
{
"$group": {
"_id": "$subject",
"count": {
"$sum": 1
}
}
}
]
results = list(db.phenotype_results.aggregate(q))
results = sorted(results, key=lambda r: r['count'], reverse=True)
except Exception as e:
traceback.print_exc(file=sys.stdout)
finally:
client.close()
return results
def main():
banditron = Banditron()
synsep = SynSep()
error_list = list()
rounds = list()
rounds_list = []
total_rounds = 100000
noise = 5000
for i in range(0, total_rounds):
rounds_list.append(i)
sample = set(random.sample(rounds_list, noise))
for t in range(0, total_rounds):
feature_vectors, true_label = synsep.generateSynSepData()
if t in sample:
true_label = random.randint(1,9)
banditron.run(feature_vectors, true_label-1)
if ((t+1)%1000) == 0:
print "%s rounds completed with error rate %s" %(str(t+1),str(banditron.error_rate))
rounds.append(banditron.number_of_rounds)
error_list.append(banditron.error_rate)
mongo_plot = MongoClient('localhost',27017)['aml']['plots']
mongo_plot.update({'_id':'synnonsep_banditron'},{'$set':{'timeStamp':datetime.datetime.now(),'rounds':rounds,'error_rate':error_list}},True)
print "Correctly classified: %s" %str(banditron.correct_classified)
print "Incorrectly classified: %s" %str(banditron.incorrect_classified)
print "Error Rate: %s" %str(banditron.error_rate)
def dump_articles():
connection = MongoClient('localhost', 27017)
db = connection.PTEST_BACKUP
results = db.crawling.find({}, {'_id': False})
"""
{
"_id" : ObjectId("54dd29d2b396811764a01330"),
"url" : "http://www.nasa.gov/pdf/55395main_12%20Earth%20Science.pdf",
"home" : "NASA",
"abstract" : "The mission of NASA's Earth Science ... and help answer qu
estions concerning many related aspects of ... forecasters in assessing particul
ate pollutio ...",
"title" : "Earth Science - NASA",
"keyword" : "aerosols+(pollution+aspects)",
"stored" : true,
"complete" : false,
"key" : "aerosols (pollution aspects)",
"hashed" : "aHR0cDovL3d3dy5uYXNhLmdvdi9wZGYvNTUzOTVtYWluXzEyJTIwRWFydGglMjBTY2llbmNlLnBkZg=="
}
"""
# upload via POST endpoint
from scripts.remote.remote import post_curling
import json
for record in results:
post_curling(_CRAWLING_POST['local'], {'resource': json.dumps(record), 'pwd': _TEMP_SECRET}, display=True)
# close the connection to MongoDB
connection.close()
def _open(self, scheme='mongodb://'):
hostname, dbname, options = self._parse_uri(scheme=scheme)
conf = self._prepare_client_options(options)
conf['host'] = hostname
env = _detect_environment()
if env == 'gevent':
from gevent import monkey
monkey.patch_all()
elif env == 'eventlet':
from eventlet import monkey_patch
monkey_patch()
mongoconn = MongoClient(**conf)
database = mongoconn[dbname]
version = mongoconn.server_info()['version']
if tuple(map(int, version.split('.')[:2])) < (1, 3):
raise NotImplementedError(
'Kombu requires MongoDB version 1.3+ (server is {0})'.format(
version))
self._create_broadcast(database, options)
self._client = database
def process(self):
client = MongoClient('localhost',44444)
db_temp_train = client['vsm_all_second']
collection1_temp_train = db_temp_train['collection1']
collection2_temp_train = db_temp_train['collection2']
collection3_temp_train = db_temp_train['collection3']
collection4_temp_train = db_temp_train['collection4']
collection5_temp_train = db_temp_train['collection5']
lineNum = 1
pat = "sa(\d)(.*)"
with open(os.path.join(self.fileroot,self.filename),"r") as fr:
for line in fr:
# 这是处理有的评论行的格式是异常的
if not re.findall(pat,line):
print("\n " + str(lineNum) + " something wrong ！")
continue
result = re.findall(pat,line)
starNum = result[0][0]
if starNum == '1':
collection1_temp_train.insert(dict(content = list(jieba.cut(result[0][1].strip()))))
elif starNum == '2':
collection2_temp_train.insert(dict(content = list(jieba.cut(result[0][1].strip()))))
elif starNum == '3':
collection3_temp_train.insert(dict(content = list(jieba.cut(result[0][1].strip()))))
elif starNum == '4':
collection4_temp_train.insert(dict(content = list(jieba.cut(result[0][1].strip()))))
elif starNum == '5':
collection5_temp_train.insert(dict(content = list(jieba.cut(result[0][1].strip()))))
print('process {0} lines'.format(lineNum),end='\r\t')
lineNum += 1
client.close()
def insertTemplate(template): dbCon = MongoClient( databasePath ) database = dbCon['allTemplates'] collection = database['template'] template['_id'] = ObjectId() collection.insert_one(template) dbCon.close()
class MongoDBPipeline(object):
collection_name = 'aqi'
items = []
def __init__(self, mongo_uri, mongo_db):
self.mongo_uri = mongo_uri
self.mongo_db = mongo_db
@classmethod
def from_crawler(cls, crawler):
return cls(
mongo_uri=crawler.settings.get('MONGO_URI'),
mongo_db=crawler.settings.get('MONGO_DATABASE')
)
def open_spider(self, spider):
print '------------connect to mongodb:', self.mongo_uri
self.client = MongoClient(self.mongo_uri)
self.db = self.client[self.mongo_db]
def close_spider(self, spider):
#insert to mongodb when close spider
print '------------insert data:',len(self.items)
print self.items
self.db[self.collection_name].insert_many(self.items)
self.client.close()
def process_item(self, item, spider):
self.items.append(item)
#self.db[self.collection_name].insert(dict(item))
return item
def sync():
client=MongoClient(MONGO_URI)
bi=client[MONGO_BI]
target=bi['product_bi_final']
mapping=bi['mapping']
for each in mapping.find({}, {'bi_cat_id': 1, 'bi_cat_name': 1, '_id': 0, 'erp_cat_id': 1}):
result = target.find({'category_id': each['bi_cat_id']}, {"updated_time": 1, '_id': 0}).sort([('updated_time', -1)]).limit(1)
try:
last_time = next(result)['updated_time']
except:
last_time = datetime(1992, 8, 24)
with mysql_con.cursor() as cur:
sql='''
SELECT
updated_time,
product_url,
sort_num,
sort_type,
sale_price,
sale_num,
rating,
product_name,
product_image,
original_price,
website_id AS dw_web_id,
comment_count,
goods_sn
FROM
website_product
WHERE
category_id =%s
AND updated_time >= "%s"
ORDER BY
updated_time ASC
'''
cur.execute(sql,(each['bi_cat_id']-cat_offset,last_time)) # 参数化调用防止SQL注入
for item in cur:
data={
'product_url':item[1],
'category_id':each['bi_cat_id'],
'erp_cat_id':each['erp_cat_id'],
'comment_count':int(item[11]),
'currency':"¥",
"original_price" : 0,
'product_image':re.findall(r'https://.+?\.jpg|http://.+?\.jpg',item[8]),
'product_name':item[7],
'rating':item[6],
'sale_num':int(item[5]),
'sale_price':int(item[4]*100),
'sort_num':item[2],
'sort_type':item[3],
'updated_time':item[0],
'dw_web_name':dw_web_name,
'goods_sn':item[12],
}
target.update_one({'goods_sn':goods_sn,'dw_web_id':item[10]+web_offset},{'$set':data},upsert=True)
print('processing {}'.format(data['product_url']))
client.close()
def getAllModules():
dbCon = MongoClient( databasePath )
database = dbCon['mibModules']
posts = database['mib']
output = posts.find()
dbCon.close()
return output
def get_annos(ip, port, user, pw, corpus, scheme, annotator, roles):
# connecting the database
client = MongoClient(host=ip, port=port, username=user, password=pw)
# navigating to the database collections to gather relevant documents
databases = client.list_database_names()
if not corpus in databases:
print('{} not found in databases'.format(corpus))
mongo_schemes = get_docs_by_prop(scheme, 'name', corpus, scheme_collection, client)
if not mongo_schemes:
print('no entries with scheme {} found'.format(scheme))
exit()
mongo_annotators = get_docs_by_prop(annotator, 'name', corpus, annotator_collection, client)
if not mongo_annotators:
print('no entries for annotator {} found'.format(scheme))
exit()
mongo_roles = get_docs_by_prop(roles, 'name', corpus, role_collection, client)
if not mongo_roles:
print('no entries for role {} found'.format(scheme))
exit()
mongo_annos = get_annotation_docs(mongo_schemes, mongo_annotators, mongo_roles, corpus, annotation_collection,
client)
# getting the annotation data and the session name
data = []
for ma in mongo_annos:
ad = get_docs_by_prop(ma['data_id'], '_id', corpus, annotation_data_collection, client)
s = get_docs_by_prop(ma['session_id'], '_id', corpus, session_collection, client)
data.append((ad, s))
return data
def data2store(dbname, data): client = MongoClient() db = client.get_database(dbname) collection = db.data data['date'] = datetime.datetime.utcnow() result = collection.insert(data) return result
class MyOffsiteMiddleware(OffsiteMiddleware):
def __init__(self, *args, **kwargs):
super(MyOffsiteMiddleware, self).__init__()
self.client = None
self.db = None
self.link_collection = None
def spider_opened(self, spider):
super(MyOffsiteMiddleware, self).spider_opened(spider)
dbname = settings.MONGO_DB['name']
collection_outlinks = settings.MONGO_DB['outlink_collection']
self.client = MongoClient()
self.db = self.client[dbname]
collection = self.db[collection_outlinks][spider.collection_name]
if collection.name in self.db.collection_names():
self.db.drop_collection(collection.name)
self.link_collection = collection
def __del__(self):
if self.client is not None:
self.client.close()
def should_follow(self, request, spider):
ans = super(MyOffsiteMiddleware, self).should_follow(request, spider)
if not ans:
lnk = WalkerItem()
lnk['status'] = ''
lnk['parent'] = request.headers.get('Referer', '')
lnk['response_hash'] = ''
lnk['type'] = ''
lnk['page'] = request.url
self.link_collection.insert(dict(lnk))
return ans
def __init__(self, hosts):
MockClientBase.__init__(self)
MongoClient.__init__(
self,
hosts,
replicaSet=MOCK_RS_NAME,
_pool_class=MockPool)
def getTableList(uri, internalTables=False, **kwargs):
"""
Get a list of known databases, each of which has a list of known
collections from the database. This is of the form [{'database':
(database 1), 'tables': [{'table': (collection 1)}, {'table':
(collection 2)}, ...]}, {'database': (database 2), 'tables': [...]},
...]
:param uri: uri to connect to the database.
:param internaltables: True to return tables about the database itself.
Ignored for Mongo.
:returns: A list of known collections.
"""
conn = MongoClient(uri)
databaseName = base.databaseFromUri(uri)
if databaseName is None:
databaseNames = conn.database_names()
else:
databaseNames = [databaseName]
results = []
for name in databaseNames:
database = conn[name]
results.append({
'database': name,
'tables': [{'table': collection, 'name': collection}
for collection in database.collection_names(False)]
})
return results
def remove_peer(peer):
host=(socket.gethostname())
c = MongoClient(host, 27017)
db=c.tejo
status=db.status
status.remove({'peer':peer})
c.close()
def check(self, resource, project_id, timestamp, value):
client = MongoClient(self.uri)
collection = client.log_service.quotas
conditions = {'resource':resource, 'project_id':project_id, 'timestamp':timestamp, 'value':value}
is_saved = collection.find(conditions).count() > 0
client.close()
return is_saved
def getAllDaily():
# Here we attempt to retrieve the data from namistt
daily = []
most_recent_daily = None
print "Attempting to retrieve all daily data"
months = ["January", "February", "March", "April", "June", "July", "August", "September",
"October", "November", "December"]
base_url = "http://www.namistt.com/DocumentLibrary/Market%20Reports/Daily/Norris%20Deonarine%20NWM%20Daily%20Market%20Report%20-"
for year in range(2012, 2015):
for month in months:
for day in range(1, 33):
d = retrieveDaily(base_url, str(day), month, str(year))
if d:
print d
daily.extend(d)
most_recent_daily = d
else:
print "Data not found"
# Now that we have collected all of the availible daily data, we will
# attempt to log the data into the database
print "Finished pulling data from namistt, begining to log the data"
try:
client = MongoClient("mongodb://*****:*****@ds043057.mongolab.com:43057/heroku_app24455461")
db = client.get_default_database()
db.drop_collection("daily")
db.drop_collection("dailyRecent")
num_stored = storeDaily(db, daily)
storeMostRecentDaily(db, most_recent_daily)
print str(num_stored) + " stored in database for daily records"
except:
print "Error in accessing database"
def processAdjustCube(countryThisRoundAdjust):
global adjustDict
global countryAdjust
client = MongoClient()
db = client['login_history']
col = db['col_adjust']
today = datetime.datetime.now().date()
ds = datetime.datetime(*(today.timetuple()[:6]))
for country,adjustList in countryThisRoundAdjust.iteritems():
intervallist = adjustList[0]
numlist = adjustList[1]
tmpResult = col.update({'recordtime':ds,'targetCountry':country},
{'$set':{'dateintervalList':intervallist,'numlist':numlist}},
upsert = True)
print tmpResult
for country,adjustList in countryThisRoundAdjust.iteritems():
countrydetail = adjustDict.get(country,dict())
intervallist = adjustList[0]
numlist = adjustList[1]
for index,interval in enumerate(intervallist):
tmplist = countrydetail.get(interval,list())
tmplist.append(numlist[index])
countrydetail[interval] = tmplist
adjustDict[country] = countrydetail
getCountryAdjust()
client.close()
def getDoubanBasic(doubanID):
coll = MongoClient()[DB][DoubanBasic]
cur = coll.find({'id': doubanID})
if cur.count() > 0:
return cur[0]
else:
return None
def get_all_from_mongo(dataset):
cl = MongoClient('localhost', 27018)
cl = cl['lSSVM']['base']
exps = []
for meta in cl.find({'dataset_name': dataset}):
exps.append(meta)
return exps
from src.app import app
from pymongo import MongoClient
from src.config import DBURL
from bson.json_util import dumps
import re
from src.helpers.errorHandler import errorHandler, Error404
#Connection to Mongo
client = MongoClient(DBURL)
db = client.get_database()
@app.route("/", methods=["GET"])
#return welcome message
def Welcome():
return dumps("Welcome to the API - Chat Sentiment Analysis Service")
@app.route("/chats",methods=["GET"])
# return all the different chats in the dataset
def getChats():
chat_list = db.chats.find({},{"_id":0,"chat_id":1,"chat_name":1})
return dumps(chat_list)
@app.route("/users",methods=["GET"])
#return all usernames in the dataset
def getUsers():
user_list = db.users.find({},{"_id":0,"user_id":1,"user_name":1})
return dumps(user_list)
@app.route("/users/<name>",methods=["GET"])
@errorHandler
# return messages for an specific username
return order_result['id']
def min_profit_buy(size):
ticker = client.get_product_ticker(product_id='BTC-USD')
ask = ticker['ask']
min_ask = float(ask) * .98
order_result = buy(round(min_ask, 2), size)
logging.info(order_result)
logging.info('USD Balance: {0}'.format(get_balance(accounts, 'USD')))
logging.info('BTC Balance: {0}'.format(get_balance(accounts, 'BTC')))
if __name__ == "__main__":
mongo_client = MongoClient('mongodb://localhost:27017/')
db = mongo_client.cryptocurrency_database
BTC_collection = db.BTC_collection
market_fee = 0.25 # coinbase per-transaction fee in dollars
min_profit_margin = .15 # percent change to perform a buy/sell
bankroll = 500 # USD to begin trade with
class MyWebsocketClient(gdax.WebsocketClient):
def __init__(self):
super(MyWebsocketClient, self).__init__()
self.url = STREAM_URL
self.message_type = "subscribe"
self.channels = ['ticker']
self.products = ["BTC-USD"]
self.mongo_collection = BTC_collection
import pymongo
import pprint
from pymongo import MongoClient
client = MongoClient('localhost', 27017)
db1 = client['egoNetwork']['tweet_data_all_user']
db2 = client['human']['all'] # 1
db3 = client['human']['each']
db4 = client['human']['oneinterest']
db5 = client['dataAnalyzedByPython']['dataAnalyzed']
db6 = client['human']['friendinterest']
for name in db2.find():
friend = []
for data in db1.find({"user_screen_name" : name['name']}):
if ( ( data['friend_screen_name'] != "" ) and data['friend_screen_name'] not in friend):
friend.append(data['friend_screen_name'])
for data in db1.find({"friend_screen_name" : name['name']}):
if ( ( data['user_screen_name'] != "" ) and data['user_screen_name'] not in friend):
friend.append(data['user_screen_name'])
db3.insert_one({"name" : name['name'], "friend" : friend})
count += 1
print(str(count) + " / 14951")
parser.add_argument(
'-d',
'--date_start',
help='The day of the month to start looking for duplicates',
default=21)
args = vars(parser.parse_args())
DB_NAME = args['db_name']
DB_URL = args['db_url']
start_date = datetime.datetime(args['year_start'], args['month_start'],
args['date_start'])
print("Connecting to MongoClient")
# client = MongoClient(DB_HOST, DB_PORT)
client = MongoClient(DB_URL)
db = client[DB_NAME]
print("Authenticating to MongoClient")
# db.authenticate(DB_USER, DB_PASS)
print("Authenticated to MongoClient")
games_collection = db.games
found_gd2_ids = {}
duplicates = 0
for game in games_collection.find({"date": {"$gt": start_date}}).sort("date"):
print(game["gd2_id"])
gd2_id = game["gd2_id"]
if found_gd2_ids.get(gd2_id) is None:
found_gd2_ids[gd2_id] = gd2_id
continue
import numpy as np
import faiss
from pymongo import MongoClient
client = MongoClient()
mongodb = client.face_recognition
d = 128
nb = 100000 # Database size
nq = 10000 # no. of queries
np.random.seed(1234)
xb = np.random.random((nb, d)).astype('float32')
xb[:, 0] += np.arange(nb) / 1000.
xq = np.random.random((nq, d)).astype('float32')
xq[:, 0] += np.arange(nq) / 1000.
index = faiss.IndexFlatL2(d) # build the index
print(index.is_trained)
index.add(xb) # add vectors to the index
print(index.ntotal)
# Searching
k = 4 # we want to see 4 nearest neighbors
D, I = index.search(xb[:5], k) # sanity check
print(I)
print(D)
D, I = index.search(xq, k) # actual search
print(I[:5]) # neighbors of the 5 first queries
print(I[-5:])
from pymongo import MongoClient
from utils.mongo_data import MONGO_HOST
client = MongoClient(MONGO_HOST)
db = client.myDigitalFootprints
for doc in db.fbUSers.find({}):
print(doc)
import plotly.graph_objects as go
import plotly as py
#%%
py.offline.init_notebook_mode(connected=True)
#%% [markdown]
# ## Getting Data From Mongo
#%%
# Set up MongoDB Client
client_mqtt = mqtt.Client()
# Set up NodeMCU Client
client_mongo = MongoClient('192.168.1.5')
temp_database = client_mongo['temp_collect']
data_1 = temp_database['Temperature_1']
data_2 = temp_database['Temperature_2']
#%% [markdown]
# # Temperature of Fixed Zone (1) - Every Sensor Plot
#%%
#Time : 2pm - 3pm , AC Temperature : 26, Date : 18/7/19
tmp_1 = []
tmp_2 = []
tmp_3 = []
avg = []
#specifying every n'th minute
time = 3
# Could not geolocate
return None, None
if __name__ == '__main__':
EVENTREGISTRY_API_KEY = os.environ["EVENTREGISTRY_API_KEY"]
if (os.environ.get('MONGODB_URI')):
db_name = os.environ.get('MONGODB_URI').split("/")[3]
db_uri = os.environ.get('MONGODB_URI')
else:
db_name = 'geochallenge'
db_uri = 'mongodb://localhost:27017/geochallenge'
db = MongoClient(db_uri)[db_name]
inserted_questions = 0
questions = db.questions
questions.create_index("uri", unique=True)
dateStart = datetime.date.today() + datetime.timedelta(days=-30)
dateEnd = datetime.date.today()
er = EventRegistry(apiKey=EVENTREGISTRY_API_KEY)
q = QueryEventsIter(dateStart=dateStart,
dateEnd=dateEnd,
lang="eng",
minArticlesInEvent=int(random.random() * 250))
for event in q.execQuery(
er,
sortBy="date",
returnInfo=ReturnInfo(eventInfo=EventInfoFlags(imageCount=1)),
#!/usr/bin/env python3
# encoding:utf-8
import sys
import options
from pymongo import MongoClient
conn = MongoClient()
adb = conn.anwen
adb.authenticate(options.db['username'], options.db['password'])
def fix():
n = 1
for i in adb.Hit_Col_v2.find().sort('_id', 1):
i['id'] = n
i['hitnum'] = 1
adb.Hit_Col.insert(i)
n += 1
fix()
from pymongo import MongoClient
client = MongoClient(
'34.122.175.106',
username='******',
password='******'
)
db = client.test
productList = db.product.find()
for product in productList:
print(product)
def put_hospital_data():
s_d = request.form.get('S_D')
malaria = request.form.get('MALARIA')
location = request.form.get('LOCATION')
s_m = request.form.get('S_M')
chickengunia = request.form.get('CHICKENGUNIA')
viral_fever = request.form.get('VIRAL_FEVER')
flu = request.form.get('FLU')
tuberculosis = request.form.get('TUBERCULOSIS')
dengue = request.form.get('DENGUE')
diarrohea = request.form.get('DIARROHEA')
typhoid = request.form.get('TYPHOID')
cholera = request.form.get('CHOLERA')
jaundice = request.form.get('JAUNDICE')
print(s_d)
print("hi" + malaria)
result = requests.get(
'http://api.airvisual.com/v1/nearest?lat=19.0728300&lon=72.8826100&key=yGQXyQuzsJ6nKiyC3'
).json()
aqi = result['data']['current']['pollution']['aqius']
temp = result['data']['current']['weather']['tp']
rain = 0
if 'pop' in result['data']['current']['weather']:
rain = result['data']['current']['weather']['pop']
print(str(aqi) + " " + str(temp) + " " + str(rain))
row = [
location, s_d, s_m, malaria, dengue, chickengunia, viral_fever, flu,
tuberculosis, diarrohea, typhoid, cholera, jaundice, temp, rain, aqi,
'1'
]
rowx = row
print('x')
print(row)
bottle_list = []
f = open('../regions', "r")
lines = f.readlines()
print(lines)
f.close()
cnt = 0
for i in range(len(lines)):
print(lines[i][:-1].lower() + " " + location.lower())
if lines[i][:-1].lower() == location.lower():
cnt = i
break
cnt = cnt + 1
with open('../preparing_data/raw/REFINEDDATA/temp/temp.csv', 'rb') as b:
bottles = csv.reader(b)
bottle_list.extend(bottles)
line_to_override = {cnt: row}
with open('../preparing_data/raw/REFINEDDATA/temp/temp.csv', 'wb') as b:
writer = csv.writer(b)
for line, row in enumerate(bottle_list):
data = line_to_override.get(line, row)
writer.writerow(data)
filename = '../preparing_data/raw/REFINEDDATA/' + location.lower() + '.csv'
content = open(filename, 'r')
linec = content.readline()
linec = linec[:-1]
data = linec.split(',')
list1 = data[3:13]
list2 = data[13:]
n = len(list2) - 3
rowx = rowx[:-1]
for j in range(n):
rowx.append(0)
with open(filename, 'ab') as fp:
a = csv.writer(fp, delimiter=',')
print('l')
print(rowx)
data = rowx
a.writerow(data)
client = MongoClient()
db = client.spateaugur
collectionname = location.lower() + '_refined'
print(collectionname)
document = db[collectionname].find_one()
listaccess = document.keys()
listremove = [
'LOCALITY', 'S_D', '_id', 'S_M', "MALARIA", "DENGUE", "CHICKENGUNIA",
"VIRAL_FEVER", "FLU", "TUBERCULOSIS", "DIARROHEA", "TYPHOID",
"CHOLERA", "JAUNDICE", "TEMP_C", "RAIN_MM", "AQI"
]
listaccess = list(set(listaccess).difference(set(listremove)))
print("List >>>>")
print(listaccess)
data = {
"LOCALITY": rowx[0],
"S_D": int(rowx[1]),
"S_M": int(rowx[2]),
"MALARIA": int(rowx[3]),
"DENGUE": int(rowx[4]),
"CHICKENGUNIA": int(rowx[5]),
"VIRAL_FEVER": int(rowx[6]),
"FLU": int(rowx[7]),
"TUBERCULOSIS": int(rowx[8]),
"DIARROHEA": int(rowx[9]),
"TYPHOID": int(rowx[10]),
"CHOLERA": int(rowx[11]),
"JAUNDICE": int(rowx[12]),
"TEMP_C": int(rowx[13]),
"RAIN_MM": int(rowx[14]),
"AQI": int(rowx[15])
}
for i in range(len(listaccess)):
data[listaccess[i]] = 0
print('Data: >>>>')
print(data)
result = db[collectionname].insert_one(data)
print(result)
print("Data2 >>>>>")
print(data)
#os.system('python ../model_retrain/trainall.py')
#os.system('python ../model_use/useall.py')
return 'done'
from pymongo import MongoClient
from gh_blog.gh_blog.settings import MONGO_URI
client = MongoClient(MONGO_URI)
gh = client.get_database('gh')
for post in gh.blogs.find():
print(post)
def __init__(self, source_db, target_db, auth_file):
self.authorize(auth_file)
self.client = MongoClient()
self.users = self.client[source_db]['users']
self.tweets = self.client[target_db]['tweets']
self.error = 0
fazer algumas atualizações Modificado em 03 de abril de 2017 por Vitor Mazuco ([email protected]) """ import sys try: from pymongo import MongoClient except: sys.exit("[!] Por favor, intale a biblioteca sqlalchemy com o comando: sudo apt-get install python-pymongo") client = MongoClient("127.0.0.1") db=client["devops"] db.fila.update({"_id":2},{"$set":{"servico":"Linux"}}) db.fila.update({"_id":4},{"$set":{"servidor":"192.168.1.100"}} ) for r in db.fila.find(): print r
from flask import Flask, render_template, jsonify, request
app = Flask(__name__)
import requests
from bs4 import BeautifulSoup
from pymongo import MongoClient # pymongo를 임포트 하기(패키지 인스톨 먼저 해야겠죠?)
client = MongoClient('localhost', 27017) # mongoDB는 27017 포트로 돌아갑니다.
db = client.dbsparta # 'dbsparta'라는 이름의 db를 만듭니다.
## HTML을 주는 부분
@app.route('/')
def home():
return render_template('index.html')
@app.route('/memo', methods=['GET'])
def listing():
# 1. 모든 document 찾기 & _id 값은 출력에서 제외하기
articles = list(db.memos.find(projection={'_id': False}))
print(articles)
# 2. articles라는 키 값으로 영화정보 내려주기
return jsonify({'result':'success', 'articles': articles})
## API 역할을 하는 부분
@app.route('/memo', methods=['POST'])
def saving():
# 1. 클라이언트로부터 데이터를 받기
url = request.form['url']
comment = request.form['comment']
print(url, comment)
__author__ = 'ASH975'
import pymongo
from pymongo import MongoClient
from tools.mongo_data import device_default, user_default
client = MongoClient('127.0.0.1', 27017)
db_aircleaner = client.aircleaner
collection_device_default = db_aircleaner.device_default
collection_device_current = db_aircleaner.device_current
collection_user_default = db_aircleaner.user_default
collection_user_current = db_aircleaner.user_current
# collection_device_default.insert_many(device_default.documents)
collection_user_default.insert_many(user_default.documents)
import requests
import re
import time
from urllib.parse import urljoin
from bs4 import BeautifulSoup
import numpy as np
import pandas as pd
from pymongo import MongoClient
client = MongoClient("localhost", 27017)
db = client.category
collection = db.category_collection
collection.delete_many({})
all_list=[]
def first(target_url):
r = requests.get(target_url)
bs = BeautifulSoup(r.text, "lxml")
return bs
def second(Soup):
pass_name = []
pass_url = []
for b in Soup.find_all("ul",class_="sub_categories"):
for b1 in b.find_all("a"):
href1 = b1.get("href")
path1 = urljoin(cookpad_url, href1)
ex1 = b1.text
pass_name.append(ex1)
pass_url.append(path1)
#!/usr/bin/python
# -*- coding: UTF8 -*-
# @See http://www.python.org/dev/peps/pep-0263/
import json
from pymongo import MongoClient
connection = MongoClient('localhost', 27017)
db = connection.candidate_analyzer
data_db = db.data
output = {}
for element in data_db.find({}).sort('streams', -1).limit(100):
for tag in element['data']['response']['tags']:
if tag['name'] in output and tag['confidence'] > 0.7:
output[tag['name']] = output[tag['name']] + 1
elif tag['confidence'] > 0.7:
output[tag['name']] = 1
i = 0
print '\nMost viewed videos:'
for key, value in sorted(output.iteritems(),
key=lambda (k, v): (v, k),
reverse=True):
print '%s: %s' % (key, value)
i = i + 1
if i == 20:
break
output = {}
for element in data_db.find({}).sort('streams', 1).limit(100):
def get_account_performance(start_date, end_date):
tmp_mongo_connstr = 'mongodb://*****:*****@10.0.1.2/client-gmi?authMechanism=SCRAM-SHA-1'
tmp_mongo_db = 'client-gmi'
mongoClient = MongoClient(tmp_mongo_connstr)
db = mongoClient[tmp_mongo_db]
collection = db.accountsummarycollection
accountdata_list_out = []
for accountdata in collection.find(
{'Batchid': {
'$gte': start_date,
'$lte': end_date
}}):
if (accountdata['SummaryDetailFlag'] == 'D'
or (accountdata['SummaryDetailFlag'] == 'S'
and accountdata['AccountType'] == '9Z')):
quotes_context = {
'FCM':
choose_fcm_name(accountdata['FCM']),
'SummaryDetailFlag':
accountdata['SummaryDetailFlag'],
'Ccy':
accountdata['Ccy'],
'Firm':
accountdata['Firm'],
'Office':
accountdata['Office'],
'Account':
accountdata['Account'],
'Batchid':
accountdata['Batchid'],
'AccountType':
accountdata['AccountType'],
'Commission':
round(accountdata['Commission'], 2),
'TotalFees':
round(accountdata['TotalFees'], 2),
'TransactionsCommissionsFees':
round(accountdata['TransactionsCommissionsFees'], 2),
'TradedQuantityBuy':
round(accountdata['TradedQuantityBuy'], 2),
'TradedQuantitySell':
round(accountdata['TradedQuantitySell'], 2),
'CurrentPosition':
round(accountdata['CurrentPosition'], 2),
'TotalEquity':
round(accountdata['TotalEquity'], 2),
'CurrentOTE':
round(accountdata['CurrentOTE'], 2),
'CurrentOV':
round(accountdata['CurrentOV'], 2),
'ChangeInAV_At_MD_Converted':
round(accountdata['ChangeInAV_At_MD_Converted'], 2),
'ConvertedAccountValueAtMarket':
round(accountdata['ConvertedAccountValueAtMarket'], 2),
'ConvertedPriorAccountValueAtMarket':
round(accountdata['ConvertedPriorAccountValueAtMarket'], 2),
'ConvertedChangeInAccountValueAtMarket':
round(accountdata['ConvertedChangeInAccountValueAtMarket'], 2),
'InitialMarginRequirement':
round(accountdata['InitialMarginRequirement'], 2),
'MaintenanceMarginRequirement':
round(accountdata['MaintenanceMarginRequirement'], 2),
'MarginExcess':
round(accountdata['MarginExcess'], 2),
'SecurityMasterID':
accountdata['SecurityMasterID'],
'SectorId':
accountdata['SectorId'],
'Sector':
accountdata['Sector'],
'SecurityMasterDesc':
accountdata['SecurityMasterDesc'],
}
accountdata_list_out.append(quotes_context)
return accountdata_list_out
def open_spider(self, spider):
self.client = MongoClient(self.mongo_uri)
self.db = self.client[self.mongo_db]
import datetime
from config import *
import plotly.plotly as py
import plotly.tools as tls
from dotenv import load_dotenv
from pymongo import MongoClient
from os.path import join, dirname
# ENV
dotenv_path = join(dirname(__file__), '.env')
load_dotenv(dotenv_path)
# DB
if os.environ['USE_DB'] == 1:
print("Using mongo db")
client = MongoClient('localhost', 32769)
db = client['cwhelper']
offers_collection = db.offers
offers_collection = db.offers
# Ploty conf
tls.set_credentials_file(username=os.environ['PLOTLY_USER'],
api_key=os.environ['PLOTLY_PASS'])
tls.set_config_file(world_readable=True, sharing='public')
tls.set_credentials_file(stream_ids=os.environ['PLOTLY_STREAM_IDS'].split(','))
stream_ids = tls.get_credentials_file()['stream_ids']
s0 = py.Stream(stream_ids[18])
s1 = py.Stream(stream_ids[19])
s2 = py.Stream(stream_ids[20])
s3 = py.Stream(stream_ids[21])
s4 = py.Stream(stream_ids[22])
def plot_from_db(action,
db,
volt_collection,
tag_collection,
port=27017,
host='localhost',
ndevices=3,
offset=0):
client = MongoClient(port=port, host=host)
database = client[db]
tag_collection = database[tag_collection]
volt_collection = database[volt_collection]
try:
if volt_collection.count_documents(
{}) + tag_collection.count_documents({}) < 2:
raise CollectionError(
'Collection not found, please check names of the collection and database'
)
except CollectionError as e:
print(e.message)
ntags = tag_collection.count_documents({'tag': action})
n = 1
# 用于查看几号设备的图
start = 3
title = config['volt_collection'][6:] + "" + action + "_cwt"
fig = plt.figure(title, figsize=(6, 8))
fig.suptitle(action + "_cwt")
# plot the data that is of a certain action one by one
for tag in tag_collection.find({'tag': action}):
# inittime termtime
inittime, termtime = tag['inittime'] - offset, tag[
'inittime'] - offset + 30
# get the arrays according to which we will plot later
times, volts = {}, {}
for i in range(1, ndevices + 1):
times[i] = []
volts[i] = []
for volt in volt_collection.find(
{'time': {
'$gt': inittime,
'$lt': termtime
}}):
device_no = int(volt['device_no'])
v = volt['voltage']
t = volt['time']
times[device_no].append(t)
volts[device_no].append(v)
style.use('default')
colors = ['r', 'b', 'g', 'c', 'm']
subtitle = ['A', 'B', 'C', 'D', 'E', 'F', 'G']
base = ntags * 100 + 10
# plot, add_subplot(211)将画布分割成2行1列，图像画在从左到右从上到下的第1块
ax = fig.add_subplot(base + n)
plt.subplots_adjust(hspace=0.5) # 函数中的wspace是子图之间的垂直间距，hspace是子图的上下间距
ax.set_title("Person" + subtitle[n - 1] + ": " +
timeToFormat(inittime + offset) + " ~ " +
timeToFormat(termtime + offset))
# 自定义y轴的区间范围，可以使图放大或者缩小
ax.set_ylim(0, 0.9)
ax.set_ylabel('Frequency')
for i in range(start, start + 1):
ax.plot(times[i],
volts[i],
label='device_' + str(i),
color=colors[i - 1],
alpha=0.9)
# gaus1、cgau8 #gaus1、dmey、mexh、cgau1、fbsp、cmor cgau8、morl、shan
wavename = "cgau8"
totalscal = len(times[i])
fc = pywt.central_frequency(wavename) # 中心频率
cparam = 2 * fc * totalscal
scales = cparam / np.arange(totalscal, 1, -1)
volts[i] = [x * 100 for x in volts[i]]
print(volts[i])
cwtmatr, freqs = pywt.cwt(volts[i], scales, wavename,
1 / 70) # 最后参数用于计算将尺度转换为实际频率
# cwtmatr, freqs = pywt.cwt(volts[i], np.arange(70, 100), 'cgau8', 1 / 70)
ax.contourf(times[i], freqs,
abs(cwtmatr)) #绘制等高线，得到的图两边高，中间一条线，表示频率一样？
# ax.contourf(times[i], freqs, cwtmatr.real, cmap=plt.cm.hot) # 绘制热力图
# if n == 1:
# ax.legend(loc='upper right')
# if n == ntags:
# ax.set_xlabel('Time')
n += 1
# 以第一个设备的时间数据为准，数据的每1/10添加一个x轴标签
xticks = []
xticklabels = []
length = len(times[1])
interval = length // 10 - 1
for i in range(0, length, interval):
xticks.append(times[1][i])
xticklabels.append(timeToSecond(times[1][i] + offset))
ax.set_xticks(xticks) # 设定标签的实际数字，数据类型必须和原数据一致
ax.set_xticklabels(
xticklabels, rotation=15) # 设定我们希望它显示的结果，xticks和xticklabels的元素一一对应
plt.show()
mqtt_control_client.message_callback_add(v_things[n]['topic'] + "/" + in_control_suffix,
self.on_message_in_control_vThing)
mqtt_control_client.subscribe(v_things[n]['topic'] + '/' + in_control_suffix)
# main
if __name__ == '__main__':
MAX_RETRY = 3
thing_visor_ID = os.environ["thingVisorID"]
# Mongodb settings
time.sleep(1.5) # wait before query the system database
db_name = "viriotDB" # name of system database
thing_visor_collection = "thingVisorC"
db_IP = os.environ['systemDatabaseIP'] # IP address of system database
db_port = os.environ['systemDatabasePort'] # port of system database
db_client = MongoClient('mongodb://' + db_IP + ':' + str(db_port) + '/')
db = db_client[db_name]
tv_entry = db[thing_visor_collection].find_one({"thingVisorID": thing_visor_ID})
valid_tv_entry = False
for x in range(MAX_RETRY):
if tv_entry is not None:
valid_tv_entry = True
break
time.sleep(3)
if not valid_tv_entry:
print("Error: ThingVisor entry not found for thing_visor_ID:", thing_visor_ID)
exit()
try:
def get_gmi_fees(start_date, end_date):
tmp_mongo_connstr = 'mongodb://*****:*****@10.0.1.2/client-gmi?authMechanism=SCRAM-SHA-1'
tmp_mongo_db = 'client-gmi'
mongoClient = MongoClient(tmp_mongo_connstr)
db = mongoClient[tmp_mongo_db]
collection = db.accountdatacollection
accountdata_list_out = []
account_summary = {}
office_summary = {}
for accountdata in collection.find(
{'Batchid': {
'$gte': start_date,
'$lte': end_date
}}):
acct_key = (accountdata['FCM'], accountdata['Office'],
accountdata['Account'])
if acct_key in account_summary:
account_summary[acct_key]['TradedQuantityBuy'] += accountdata[
'TradedQuantityBuy']
account_summary[acct_key]['TradedQuantitySell'] += accountdata[
'TradedQuantitySell']
account_summary[acct_key]['Commission'] += accountdata[
'Commission']
account_summary[acct_key]['ClearingFees'] += accountdata[
'ClearingFees']
account_summary[acct_key]['ExchangeFees'] += accountdata[
'ExchangeFees']
account_summary[acct_key]['NFAFees'] += accountdata['NFAFees']
account_summary[acct_key]['BrokerageFees'] += accountdata[
'BrokerageFees']
account_summary[acct_key]['TradeProcessingFees'] += accountdata[
'TradeProcessingFees']
account_summary[acct_key]['CBOT_Globex_Fee'] += accountdata[
'CBOT_Globex_Fee']
account_summary[acct_key]['CME_Globex_Fee'] += accountdata[
'CME_Globex_Fee']
account_summary[acct_key]['Give_In_Fee'] += accountdata[
'Give_In_Fee']
account_summary[acct_key]['TotalFees'] += accountdata['TotalFees']
account_summary[acct_key]['SumOfFeesAndCommission'] += accountdata[
'Commission'] + accountdata['TotalFees']
else:
account_summary[acct_key] = accountdata
account_summary[acct_key]['SumOfFeesAndCommission'] = 0
for accountdata in collection.find(
{'Batchid': {
'$gte': start_date,
'$lte': end_date
}}):
office_key = (accountdata['FCM'], accountdata['Office'])
if office_key in office_summary:
office_summary[office_key]['TradedQuantityBuy'] += accountdata[
'TradedQuantityBuy']
office_summary[office_key]['TradedQuantitySell'] += accountdata[
'TradedQuantitySell']
office_summary[office_key]['Commission'] += accountdata[
'Commission']
office_summary[office_key]['ClearingFees'] += accountdata[
'ClearingFees']
office_summary[office_key]['ExchangeFees'] += accountdata[
'ExchangeFees']
office_summary[office_key]['NFAFees'] += accountdata['NFAFees']
office_summary[office_key]['BrokerageFees'] += accountdata[
'BrokerageFees']
office_summary[office_key]['TradeProcessingFees'] += accountdata[
'TradeProcessingFees']
office_summary[office_key]['CBOT_Globex_Fee'] += accountdata[
'CBOT_Globex_Fee']
office_summary[office_key]['CME_Globex_Fee'] += accountdata[
'CME_Globex_Fee']
office_summary[office_key]['Give_In_Fee'] += accountdata[
'Give_In_Fee']
office_summary[office_key]['TotalFees'] += accountdata['TotalFees']
office_summary[office_key][
'SumOfFeesAndCommission'] += accountdata[
'Commission'] + accountdata['TotalFees']
else:
office_summary[office_key] = accountdata
office_summary[office_key][
'Office'] = "Summary " + choose_fcm_name(
accountdata['FCM']) + " " + accountdata['Office']
office_summary[office_key]['SumOfFeesAndCommission'] = 0
# accountSummary
for key, value in account_summary.items():
quotes_context = {
'FCM': choose_fcm_name(value['FCM']),
'Office': value['Office'],
'Account': value['Account'],
'TradedQuantityBuy': value['TradedQuantityBuy'],
'TradedQuantitySell': value['TradedQuantitySell'],
'Commission': round(value['Commission'], 2),
'ClearingFees': round(value['ClearingFees'], 2),
'ExchangeFees': round(value['ExchangeFees'], 2),
'TransactionFees': round(value['TransactionFees'], 2),
'NFAFees': round(value['NFAFees'], 2),
'BrokerageFees': round(value['BrokerageFees'], 2),
'TradeProcessingFees': round(value['TradeProcessingFees'], 2),
'CBOT_Globex_Fee': round(value['CBOT_Globex_Fee'], 2),
'CME_Globex_Fee': round(value['CME_Globex_Fee'], 2),
'Give_In_Fee': round(value['Give_In_Fee'], 2),
'TotalFees': round(value['TotalFees'], 2),
'SumOfFeesAndCommission': round(value['SumOfFeesAndCommission'],
2),
}
print(quotes_context)
accountdata_list_out.append(quotes_context)
for key, value in office_summary.items():
quotes_context = {
'FCM': choose_fcm_name(value['FCM']),
'Office': value['Office'],
'Account': value['Account'],
'TradedQuantityBuy': value['TradedQuantityBuy'],
'TradedQuantitySell': value['TradedQuantitySell'],
'Commission': round(value['Commission'], 2),
'ClearingFees': round(value['ClearingFees'], 2),
'ExchangeFees': round(value['ExchangeFees'], 2),
'TransactionFees': round(value['TransactionFees'], 2),
'NFAFees': round(value['NFAFees'], 2),
'BrokerageFees': round(value['BrokerageFees'], 2),
'TradeProcessingFees': round(value['TradeProcessingFees'], 2),
'CBOT_Globex_Fee': round(value['CBOT_Globex_Fee'], 2),
'CME_Globex_Fee': round(value['CME_Globex_Fee'], 2),
'Give_In_Fee': round(value['Give_In_Fee'], 2),
'TotalFees': round(value['TotalFees'], 2),
'SumOfFeesAndCommission': round(value['SumOfFeesAndCommission'],
2),
}
#print(quotes_context);
accountdata_list_out.append(quotes_context)
return accountdata_list_out
def get(self):
client = MongoClient(DB_URI)
database = client[DB_NAME]
collection = database.nationalparks
return format_result(collection.find())
def active(cls, username):
client = MongoClient('mongodb://118.31.19.120:27017/')
db = client.get_database('node_club_dev')
users = db.get_collection('users')
users.find_one_and_update({"loginname": username},{"$set": {"active": True}})
return users.find_one({"loginname": username})['active']
from pymongo import MongoClient from config import MONGO_URI cli = MongoClient(MONGO_URI)
def __init__(self):
self.db = MongoClient('localhost', 8000)
self.data = self.db.base.account
def connect(dbname, host='localhost', port=27017):
""":return mongo database"""
client = MongoClient(host, port)
db = client[dbname]
return db
def __init__(self, db, collection, timeout=300):
self.client = MongoClient()
self.Client = self.client[db]
self.db = self.Client[collection]
self.timeout = timeout
