def run(self):
while True:
try:
fbid = self.queue.get()
trurl = url + '/v2.3/'
rurl = trurl + fbid
response = requests.get(rurl, params={'access_token': access_token, 'fields': 'about,category,bio,name,posts.limit(10){message}'})
data = response.json()
if 'posts' not in data:
data['posts'] = {}
data['posts']['data'] = []
document = {
'_id' : fbid,
'name' : self.getdata(data,'name'),
'about' : self.getdata(data,'about'),
'category' : self.getdata(data,'category'),
'bio' : self.getdata(data,'bio'),
'posts' : [ self.getdata(msg,'message') for msg in data['posts']['data']]
}
string = getAppendString(document)
a = self.collection.insert(string)
except:
self.queue.put(fbid)
time.sleep(2)
print fbid
self.queue.task_done()
def fetchTestpage(fbid):
global url
trurl = url + '/v2.3/'
rurl = str(trurl) + str(fbid)
response = requests.get(
rurl,
params={
'access_token': access_token,
'fields': 'about,category,bio,name,posts.limit(10){message}'
})
data = response.json()
if 'posts' not in data:
data['posts'] = {}
data['posts']['data'] = []
document = {
'_id': fbid,
'name': getdata(data, 'name'),
'about': getdata(data, 'about'),
'category': getdata(data, 'category'),
'bio': getdata(data, 'bio'),
'posts': [getdata(msg, 'message') for msg in data['posts']['data']]
}
string = getAppendString(document)
f = open('temp/' + str(string['_id']) + '.txt', 'w')
f.write(string['data'])
f.close()
def run(self):
while True:
try:
fbid = self.queue.get()
trurl = url + '/v2.3/'
rurl = trurl + fbid
response = requests.get(
rurl,
params={
'access_token':
access_token,
'fields':
'about,category,bio,name,posts.limit(10){message}'
})
data = response.json()
if 'posts' not in data:
data['posts'] = {}
data['posts']['data'] = []
document = {
'_id':
fbid,
'name':
self.getdata(data, 'name'),
'about':
self.getdata(data, 'about'),
'category':
self.getdata(data, 'category'),
'bio':
self.getdata(data, 'bio'),
'posts': [
self.getdata(msg, 'message')
for msg in data['posts']['data']
]
}
string = getAppendString(document)
a = self.collection.insert(string)
except:
self.queue.put(fbid)
time.sleep(2)
print fbid
self.queue.task_done()
def fetchTestpage(fbid):
global url
trurl = url + '/v2.3/'
rurl = str(trurl) + str(fbid)
response = requests.get(rurl, params={'access_token': access_token, 'fields': 'about,category,bio,name,posts.limit(10){message}'})
data = response.json()
if 'posts' not in data:
data['posts'] = {}
data['posts']['data'] = []
document = {
'_id' : fbid,
'name' : getdata(data,'name'),
'about' : getdata(data,'about'),
'category' : getdata(data,'category'),
'bio' : getdata(data,'bio'),
'posts' : [ getdata(msg,'message') for msg in data['posts']['data']]
}
string = getAppendString(document)
f = open('temp/' + str(string['_id']) + '.txt', 'w')
f.write(string['data'])
f.close()
