def gatherData(username):
sequence = 0
VisitedNodes = set()
queue = FifoDiskQueue("FriendsQueue")
queue.push(username)
sequence += 1
mapToRemoveDuplicates = {username: sequence}
mapUsernameSequence = {username:sequence}
anonymize(username, sequence)
while (queue.__len__()>0):
vertex = queue.pop()
edges = parseFriends(vertex)
VisitedNodes.add(vertex)
count = VisitedNodes.__len__()
if(count <= VertexLimit):
for user in edges:
if not VisitedNodes.__contains__(user):
if (mapToRemoveDuplicates.get(user) == None):
queue.push(user)
sequence += 1
mapToRemoveDuplicates.update({user: sequence})
mapUsernameSequence.update({user:sequence})
anonymize(user, sequence)
else:
exit()
print "Total Visited nodes " + str(count)
saveData(vertex, edges)
anonymizeDataset(vertex, edges, mapUsernameSequence)
def run(self):
global global_l
counter = 0
while self._stop_t == False:
counter += 1
time.sleep(round(self.client.delay - (self.client.delay / 4.), 2))
if counter % self.client.batch_size == 0 and self.client.connection_error is False:
print("sending data from thread {}:{}".format(
self.client.cls_name(), id(self.client)))
self.client.send_data(global_l[:self.client.batch_size])
global_l = global_l[self.client.batch_size:]
counter = 0
if len(global_l) > 4 * self.client.batch_size:
fq = FifoDiskQueue("{}.fifo.sql".format(self.client.name))
for obj in global_l[:self.client.batch_size * 2]:
print("SAVED: {}".format(obj))
fq.push(obj)
global_l = global_l[self.client.batch_size * 2:]
fq.close()
if (counter + 2) % self.client.batch_size == 0:
fq = FifoDiskQueue("{}.fifo.sql".format(self.client.name))
if len(fq) > 0:
for i in range(2):
for elem in fq.pull():
print("PULL ELEM", elem)
global_l.append(elem)
print("DB SIZE", len(fq))
fq.close()
def addSettings():
settings = request.form['settings']
if (settings != "null"):
setting_queue = FifoDiskQueue("settings_file")
setting_queue.push(settings.encode(encoding='UTF-8'))
setting_queue.close()
return "sucess"
return "No settings to update"
def fetch_data(self):
error = None
tags = None
try:
if self.config['RANGE']:
RANGE_NAME = self.config['SHEETNAME'] + "!" + self.config[
'RANGE']
else:
RANGE_NAME = self.config['SHEETNAME']
#call class method which return sheet data and error if permission is not there
getValueMapping = self.get_sheetvalues(self.config['SHEETID'],
RANGE_NAME)
mappingError = getValueMapping[1] # Error in fetching mapping
mappingValues = getValueMapping[0] # mapping values list
if not mappingError:
raw_data = mappingValues
# Create a JSON from data.
column_names = raw_data[0]
final_data = []
for data in raw_data[2:]:
single_data = dict()
counter = 0
for col in column_names:
single_data[col] = data[counter]
counter += 1
tags = self.get_tags()
all_data = dict()
all_data['req_data'] = single_data
all_data.update(
self.config) # merge tags with sheet each row data
#final_data.append(raw_data)
raw_data = dict()
raw_data['reqd_data'] = all_data
raw_data['tags'] = tags
q = FifoDiskQueue(
os.path.dirname(__file__) + '/../../queuedata')
q.push(json.dumps(raw_data).encode('utf-8'))
q.close()
else:
error = "No Mapping details found"
except Exception as e:
import traceback
error = "Failed to fetch mapping detials"
mappingValues = None
return error
def addTab():
drink_name = request.form['drink_name']
if(drink_name != "null"):
conn = getdbConn()
tab = FifoDiskQueue("tab_file")
drink_id = conn.execute('SELECT id FROM Drink WHERE name LIKE "' + drink_name + '" LIMIT 1;').fetchone()
if(drink_id == None):
return json.dumps({'success':False,'error':"NoDrinkInDB"}), 400, {'ContentType':'application/json'}
tab.push(str(drink_id[0]).encode(encoding='UTF-8'))
tab.close()
conn.close()
return json.dumps({'success':True, 'hello':'hey'}), 200, {'ContentType':'application/json'}
return json.dumps({'success':False,'error':"DrinkNameNull"}), 400, {'ContentType':'application/json'}
disk_buffer = FifoDiskQueue(BUFFER_FILE)
else:
# DFS / FILE
disk_buffer = LifoDiskQueue(BUFFER_FILE)
unvisited_links_in = Queue()
unvisited_links_out = Queue()
unparsed_html = JoinableQueue()
buffer_lock = threading.Lock()
first_link = dict()
first_link['url'] = start_url
first_link['parent_url'] = None
first_link['level'] = 0
# add first link to queue
disk_buffer.push(pickle.dumps(first_link))
# setups and start threads
threads = list()
for t in range(NUM_THREADS):
s = Scraper(unparsed_html, disk_buffer, buffer_lock)
s.daemon = True
s.start()
threads.append(s)
pool = multiprocessing \
.Pool(1, Parser(unparsed_html, unvisited_links_out, visited_links, visited_lock, max_levels, keyword)
.run)
def signal_handler(signal, frame):
with buffer_lock:
import os
import sys
import networkx as nx
from queuelib import FifoDiskQueue
q = FifoDiskQueue("diskFile3")
completed = False
for root, dirs, files in os.walk(sys.argv[1]):
for file in files:
print file
q.push(sys.argv[1] + file)
q.close()
print "-----------------"
while True:
q = FifoDiskQueue("diskFile3")
file1 = q.pop()
file2 = q.pop()
q.close()
if file1 and file2:
print file1
print file2
fileId1 = "_".join(file1.split("_")[1:5]).split(".")[0]
fileId2 = "_".join(file2.split("_")[1:5]).split(".")[0]
newFileId = sys.argv[2] + "_" + fileId1 + "_" + fileId2 + ".gpickle"
print newFileId
G1 = nx.read_gpickle(file1)
G2 = nx.read_gpickle(file2)
G1.add_nodes_from(G2.nodes(data=True))
G1.add_edges_from(G2.edges(data=True))
nx.write_gpickle(G1, newFileId)
def push(queue, async_spec):
queue = FifoDiskQueue(queue)
queue.push(json.dumps(async_spec).encode())
queue.close()
sched = BlockingScheduler()
APP_KEY = api_keys.APP_KEY
APP_SECRET = api_keys.APP_SECRET
OAUTH_TOKEN = api_keys.OAUTH_TOKEN
OAUTH_TOKEN_SECRET = api_keys.OAUTH_TOKEN_SECRET
twitter = Twython(APP_KEY, APP_SECRET, OAUTH_TOKEN, OAUTH_TOKEN_SECRET)
selfies = ['1.png', '2.png', '3.png']
q = FifoDiskQueue("queuefile")
if len(q) == 0:
for selfie in selfies:
q.push(selfie)
selfie = q.pop()
q.close()
@sched.scheduled_job('interval', minutes=10)
def revolve_avatar():
avatar = open("selfies/" + selfie, 'rb')
twitter.update_profile_image(image=avatar)
@sched.scheduled_job('cron', day_of_week='mon-sun', hour=5)
def evolve_banner():
twitter.update_profile_banner_image(banner=image)
def save(self, messages):
queue = FifoDiskQueue(self.database_name)
for message in messages:
queue.push(message)
queue.close()
def run(self, fn, sleep=1):
while True:
queue = FifoDiskQueue(self.database_name)
for message in self.generate_data(fn, sleep=sleep):
queue.push(message)
queue.close()
def add_job(self, s):
queue = FifoDiskQueue(self.queue)
queue.push(s.encode('latin1'))
queue.close()
def fetch_data(self):
error = None
tags = None
try:
reqData = json.loads(json.dumps(request.json)) # Converting unicodes to str and then str to dict.
formId = reqData[
'formId'] # Getting the form ID to distinguish between various template documents and mapping sheets
newReqData = reqData['data'][0] # Getting the data : [{values}]
instanceId = newReqData['instanceID'] # Getting the instance id for searching routes
newReqData = json.loads(json.dumps(newReqData)) # Getting the new data
userName = newReqData['username']
formSubmissionDate = newReqData[
'*meta-submission-date*'] # Correcting the submission date and removing the time
endIndex = formSubmissionDate.find(str('T'))
formSubmissionDate = formSubmissionDate[:endIndex]
newReqData['*meta-submission-date*'] = formSubmissionDate # Saving the corrected date in the json
myDict = {}
for k, v in newReqData.items():
if type(v) is dict:
for k1, v1 in v.items():
if k1 == "url":
# correcting the URLs
baseUrl = 'http://aggregate.cttsamagra.xyz:8080/'
indexStart = 0 # Finding the substring
indexEnd = v1.find(
":8080/") + 6 # Find the stopping point
newv1 = v1.replace(v1[indexStart:indexEnd], baseUrl)
myDict[k] = newv1
elif type(v) is float or type(v) is int:
myDict[k] = str(v)
elif type(v) is list:
myDict[k] = str(v[0]) # Converting list to str
else:
if v is None:
v = "NO_TEXT_FOUND"
myDict[k] = v
# Calculate Udise from its database and then Calculate distance from udise
calculated_distance = 'Not available' # Calculate using udise
myDict['calculated_distance'] = calculated_distance
all_data = dict()
all_data['req_data'] = myDict
all_data['FORMID'] = formId
all_data['INSTANCEID'] = instanceId
all_data['USERNAME'] = userName
all_data['FORMSUBMISSIONDATE'] = formSubmissionDate
self.raw_data = all_data
tags = self.get_tags()
all_data.update(self.config)
raw_data = dict()
raw_data['reqd_data'] = all_data
raw_data['tags'] = tags
q = FifoDiskQueue(os.path.dirname(__file__)+'/../../queuedata')
q.push(json.dumps(raw_data).encode('utf-8'))
q.close()
except Exception as e:
import traceback
error = "Failed to fetch mapping detials"
return error
def push(self, reqt):
return FifoDiskQueue.push(self, self.marshal(reqt))
