def data_download(self):
"""
function for saving data from queue into PdfData table
"""
print("Starting data download")
print("-" * 79)
with self._app.app_context():
while 1:
da_queue = FifoDiskQueue(
os.path.dirname(__file__) + '/../queuedata')
data = da_queue.pop()
da_queue.close()
if not data:
print('sleep for 10')
sleep(10)
#break
else:
raw_data = json.loads(data.decode('utf-8'))
info_log(self.logger.info, "Step1 Save into db Start",
raw_data['reqd_data'])
self._save_pdf_data(raw_data['reqd_data'],
raw_data['tags'],
raw_data['instance_id'])
info_log(self.logger.info, "Step1 Save into db End",
raw_data['reqd_data'])
def pop(queue):
queue = FifoDiskQueue(queue)
async_spec_bytes = queue.pop()
queue.close()
if async_spec_bytes:
return json.loads(async_spec_bytes.decode())
else:
return None
def run(self):
queue = FifoDiskQueue(self.queue)
while True:
if self.query_cancel():
break
qe = queue.pop()
if qe == None:
break
task = qe.decode('latin1')
os.system(task)
queue.close()
def run(self):
while True:
queue = FifoDiskQueue("{}.fifo.sql".format(self.name))
#print(len(queue))
if len(queue) > 0 and self.is_network_up():
messages = queue.pull()
response = self.send_blocks_msg(messages)
if response is None or response is not True:
self.sync_failed(response, messages)
queue.close()
time.sleep(self.DELAY)
def gatherData(username):
sequence = 0
VisitedNodes = set()
queue = FifoDiskQueue("FriendsQueue")
queue.push(username)
sequence += 1
mapToRemoveDuplicates = {username: sequence}
mapUsernameSequence = {username:sequence}
anonymize(username, sequence)
while (queue.__len__()>0):
vertex = queue.pop()
edges = parseFriends(vertex)
VisitedNodes.add(vertex)
count = VisitedNodes.__len__()
if(count <= VertexLimit):
for user in edges:
if not VisitedNodes.__contains__(user):
if (mapToRemoveDuplicates.get(user) == None):
queue.push(user)
sequence += 1
mapToRemoveDuplicates.update({user: sequence})
mapUsernameSequence.update({user:sequence})
anonymize(user, sequence)
else:
exit()
print "Total Visited nodes " + str(count)
saveData(vertex, edges)
anonymizeDataset(vertex, edges, mapUsernameSequence)
def dataDownload(self):
print("Starting data download")
print("-" * 79)
with self._app.app_context():
while 1:
q = FifoDiskQueue(os.path.dirname(__file__) + '/../queuedata')
data = q.pop()
q.close()
if not data:
print('sleep for 10')
sleep(10)
#break
else:
raw_data = json.loads(data.decode('utf-8'))
self.save_pdf_data(raw_data['reqd_data'], raw_data['tags'])
class Thingsspeak(object):
"""docstring for Thingsspeak"""
def __init__(self):
super(Thingsspeak, self).__init__()
self.retry_queue = FifoDiskQueue("/var/thingsspeak.queue")
def send(self, values):
print 'Thingspeak:', values
values = {'created_at': values['created_at'],
'key': 'MYA8R6YDMDQNM9M9',
'field1': values['target_temp'],
'field2': values['current_temp'],
'field3': values['control_value'],
'field4': values['chiller_raw'],
'field5': values['heater_raw'],
'field6': values['current_temp_2']
}
print values
postdata = urllib.urlencode(values)
response_html = self.post_data(postdata)
if response_html != '0':
self.retry()
def retry(self):
if (len(self.retry_queue) > 0):
time.sleep(15)
postdata = self.retry_queue.pop()
print "retrying: ", postdata
if self.post_data(postdata) == 0:
return
#
# 'created_at': self.time,
# 'target_temp': str(self.target_temp),
# 'current_temp': '{0:.3f}'.format(self.current_temp),
# 'control_value': '{0:.0f}'.format(self.control_value),
# 'chiller': self.chiller.get_state_str(),
# 'chiller_info': self.chiller.get_info(),
# 'heater': self.heater.get_state_str(),
# 'heater_info': self.heater.get_info()
def post_data(self, postdata):
target_url = 'https://api.thingspeak.com/update'
request = urllib2.Request(target_url, postdata)
response_html = None
try:
response = urllib2.urlopen(request, None, 5)
response_html = response.read()
response.close()
print response_html
except urllib2.HTTPError, e:
print 'Server could not fulfill the request. Error code: ' + str(e.code)
except urllib2.URLError, e:
print 'Failed to reach server. Reason: ' + str(e.reason)
def tap():
cabinet_id = request.args.get("cabinet_id")
if(cabinet_id is not None):
tab = FifoDiskQueue("tab_file")
current_drink_id = tab.pop()
if(current_drink_id != None):
conn = getdbConn()
drink_id = int(current_drink_id.decode(encoding='UTF-8'))
tab.close()
cursor = conn.execute(getSerachString(drink_id))
ingredients = []
for row in cursor:
ingredients.append({
"name": row[1],
"ratio": row[2]
})
drink = {
"drink_name": row[0],
"ingredients": ingredients
}
response = {"settings":"null","drink":drink}
return json.dumps(response)
return json.dumps({"settings": "null", "drink":"null"})
else:
return json.dumps({'success': False,'error':"Must Have Cabinet ID to access to tap"}), 400, {'ContentType':'application/json'}
def update_all(root, image_label, cam, data_label):
update_image(image_label, cam)
interval = time.time() - start_time
#print interval
if(interval >2 ):
saveimage(cam.read())
global q
q = FifoDiskQueue("queuefile")
#q.push(b
global start_time
start_time = time.time()
#update_fps(fps_label)
root.after(2, func=lambda: update_all(root, image_label, cam,data_label))
def addSettings():
settings = request.form['settings']
if (settings != "null"):
setting_queue = FifoDiskQueue("settings_file")
setting_queue.push(settings.encode(encoding='UTF-8'))
setting_queue.close()
return "sucess"
return "No settings to update"
def getTab():
tab = FifoDiskQueue("tab_file")
content = []
t = tab.pop()
while(t != None):
t = t.decode(encoding='UTF-8')
content.append(t)
t = tab.pop()
tab.close()
return json.dumps(content)
def fetch_data(self):
error = None
tags = None
try:
if self.config['RANGE']:
RANGE_NAME = self.config['SHEETNAME'] + "!" + self.config[
'RANGE']
else:
RANGE_NAME = self.config['SHEETNAME']
#call class method which return sheet data and error if permission is not there
getValueMapping = self.get_sheetvalues(self.config['SHEETID'],
RANGE_NAME)
mappingError = getValueMapping[1] # Error in fetching mapping
mappingValues = getValueMapping[0] # mapping values list
if not mappingError:
raw_data = mappingValues
# Create a JSON from data.
column_names = raw_data[0]
final_data = []
for data in raw_data[2:]:
single_data = dict()
counter = 0
for col in column_names:
single_data[col] = data[counter]
counter += 1
tags = self.get_tags()
all_data = dict()
all_data['req_data'] = single_data
all_data.update(
self.config) # merge tags with sheet each row data
#final_data.append(raw_data)
raw_data = dict()
raw_data['reqd_data'] = all_data
raw_data['tags'] = tags
q = FifoDiskQueue(
os.path.dirname(__file__) + '/../../queuedata')
q.push(json.dumps(raw_data).encode('utf-8'))
q.close()
else:
error = "No Mapping details found"
except Exception as e:
import traceback
error = "Failed to fetch mapping detials"
mappingValues = None
return error
def addTab():
drink_name = request.form['drink_name']
if(drink_name != "null"):
conn = getdbConn()
tab = FifoDiskQueue("tab_file")
drink_id = conn.execute('SELECT id FROM Drink WHERE name LIKE "' + drink_name + '" LIMIT 1;').fetchone()
if(drink_id == None):
return json.dumps({'success':False,'error':"NoDrinkInDB"}), 400, {'ContentType':'application/json'}
tab.push(str(drink_id[0]).encode(encoding='UTF-8'))
tab.close()
conn.close()
return json.dumps({'success':True, 'hello':'hey'}), 200, {'ContentType':'application/json'}
return json.dumps({'success':False,'error':"DrinkNameNull"}), 400, {'ContentType':'application/json'}
def __init__(self, filename, marshal=lambda x: x, unmarshal=lambda x: x):
FifoDiskQueue.__init__(self, filename)
self.marshal = marshal
self.unmarshal = unmarshal
def pop(self):
return self.unmarshal(FifoDiskQueue.pop(self))
# use lock to visited links so only one thread can update at a time
visited_lock = threading.Lock()
# make visited links a hashed set so there are not duplicates
# a bloom filter may improve performance with less memory
visited_links = set()
# create a queue of unvisited links added by threads as they scrape
if os.path.isdir(BUFFER_FILE):
shutil.rmtree(BUFFER_FILE)
elif os.path.isfile(BUFFER_FILE):
os.remove(BUFFER_FILE)
if int(search_type) == 1:
# BFS / DIR
disk_buffer = FifoDiskQueue(BUFFER_FILE)
else:
# DFS / FILE
disk_buffer = LifoDiskQueue(BUFFER_FILE)
unvisited_links_in = Queue()
unvisited_links_out = Queue()
unparsed_html = JoinableQueue()
buffer_lock = threading.Lock()
first_link = dict()
first_link['url'] = start_url
first_link['parent_url'] = None
first_link['level'] = 0
# add first link to queue
disk_buffer.push(pickle.dumps(first_link))
def run(self):
global global_l
counter = 0
while self._stop_t == False:
counter += 1
time.sleep(round(self.client.delay - (self.client.delay / 4.), 2))
if counter % self.client.batch_size == 0 and self.client.connection_error is False:
print("sending data from thread {}:{}".format(
self.client.cls_name(), id(self.client)))
self.client.send_data(global_l[:self.client.batch_size])
global_l = global_l[self.client.batch_size:]
counter = 0
if len(global_l) > 4 * self.client.batch_size:
fq = FifoDiskQueue("{}.fifo.sql".format(self.client.name))
for obj in global_l[:self.client.batch_size * 2]:
print("SAVED: {}".format(obj))
fq.push(obj)
global_l = global_l[self.client.batch_size * 2:]
fq.close()
if (counter + 2) % self.client.batch_size == 0:
fq = FifoDiskQueue("{}.fifo.sql".format(self.client.name))
if len(fq) > 0:
for i in range(2):
for elem in fq.pull():
print("PULL ELEM", elem)
global_l.append(elem)
print("DB SIZE", len(fq))
fq.close()
def fetch_data(self):
error = None
tags = None
try:
reqData = json.loads(json.dumps(request.json)) # Converting unicodes to str and then str to dict.
formId = reqData[
'formId'] # Getting the form ID to distinguish between various template documents and mapping sheets
newReqData = reqData['data'][0] # Getting the data : [{values}]
instanceId = newReqData['instanceID'] # Getting the instance id for searching routes
newReqData = json.loads(json.dumps(newReqData)) # Getting the new data
userName = newReqData['username']
formSubmissionDate = newReqData[
'*meta-submission-date*'] # Correcting the submission date and removing the time
endIndex = formSubmissionDate.find(str('T'))
formSubmissionDate = formSubmissionDate[:endIndex]
newReqData['*meta-submission-date*'] = formSubmissionDate # Saving the corrected date in the json
myDict = {}
for k, v in newReqData.items():
if type(v) is dict:
for k1, v1 in v.items():
if k1 == "url":
# correcting the URLs
baseUrl = 'http://aggregate.cttsamagra.xyz:8080/'
indexStart = 0 # Finding the substring
indexEnd = v1.find(
":8080/") + 6 # Find the stopping point
newv1 = v1.replace(v1[indexStart:indexEnd], baseUrl)
myDict[k] = newv1
elif type(v) is float or type(v) is int:
myDict[k] = str(v)
elif type(v) is list:
myDict[k] = str(v[0]) # Converting list to str
else:
if v is None:
v = "NO_TEXT_FOUND"
myDict[k] = v
# Calculate Udise from its database and then Calculate distance from udise
calculated_distance = 'Not available' # Calculate using udise
myDict['calculated_distance'] = calculated_distance
all_data = dict()
all_data['req_data'] = myDict
all_data['FORMID'] = formId
all_data['INSTANCEID'] = instanceId
all_data['USERNAME'] = userName
all_data['FORMSUBMISSIONDATE'] = formSubmissionDate
self.raw_data = all_data
tags = self.get_tags()
all_data.update(self.config)
raw_data = dict()
raw_data['reqd_data'] = all_data
raw_data['tags'] = tags
q = FifoDiskQueue(os.path.dirname(__file__)+'/../../queuedata')
q.push(json.dumps(raw_data).encode('utf-8'))
q.close()
except Exception as e:
import traceback
error = "Failed to fetch mapping detials"
return error
def __init__(self):
super(Thingsspeak, self).__init__()
self.retry_queue = FifoDiskQueue("/var/thingsspeak.queue")
def initializeQueue(self):
"""initialize the queue instance."""
queuedir = self.log_dir + '/b2safe.queue'
self.queue = FifoDiskQueue(queuedir)
def add_job(self, s):
queue = FifoDiskQueue(self.queue)
queue.push(s.encode('latin1'))
queue.close()
import threading
from queuelib import FifoDiskQueue
import os.path
import re
import sys
import tarfile
from mygui import *
from mytf import *
import numpy as np
from six.moves import urllib
import tensorflow as tf
tf.app.flags.DEFINE_string('image_path', '/home/ps/Documents/gui',
"""Absolute path to image file.""")
start_time = time.time()
q = FifoDiskQueue("queuefile")
file_no = 0
def addmodel():
a = askopenfilename()
print a
def run_inference_on_image(image):
#create_graph()
if not tf.gfile.Exists(image):
tf.logging.fatal('File does not exist %s', image)
image_data = tf.gfile.FastGFile(image, 'rb').read()
create_graph()
img = image
import os
import sys
import networkx as nx
from queuelib import FifoDiskQueue
q = FifoDiskQueue("diskFile3")
completed = False
for root, dirs, files in os.walk(sys.argv[1]):
for file in files:
print file
q.push(sys.argv[1] + file)
q.close()
print "-----------------"
while True:
q = FifoDiskQueue("diskFile3")
file1 = q.pop()
file2 = q.pop()
q.close()
if file1 and file2:
print file1
print file2
fileId1 = "_".join(file1.split("_")[1:5]).split(".")[0]
fileId2 = "_".join(file2.split("_")[1:5]).split(".")[0]
newFileId = sys.argv[2] + "_" + fileId1 + "_" + fileId2 + ".gpickle"
print newFileId
G1 = nx.read_gpickle(file1)
G2 = nx.read_gpickle(file2)
G1.add_nodes_from(G2.nodes(data=True))
G1.add_edges_from(G2.edges(data=True))
nx.write_gpickle(G1, newFileId)
def push(queue, async_spec):
queue = FifoDiskQueue(queue)
queue.push(json.dumps(async_spec).encode())
queue.close()
from queuelib import FifoDiskQueue
from apscheduler.schedulers.blocking import BlockingScheduler
import api_keys
sched = BlockingScheduler()
APP_KEY = api_keys.APP_KEY
APP_SECRET = api_keys.APP_SECRET
OAUTH_TOKEN = api_keys.OAUTH_TOKEN
OAUTH_TOKEN_SECRET = api_keys.OAUTH_TOKEN_SECRET
twitter = Twython(APP_KEY, APP_SECRET, OAUTH_TOKEN, OAUTH_TOKEN_SECRET)
selfies = ['1.png', '2.png', '3.png']
q = FifoDiskQueue("queuefile")
if len(q) == 0:
for selfie in selfies:
q.push(selfie)
selfie = q.pop()
q.close()
@sched.scheduled_job('interval', minutes=10)
def revolve_avatar():
avatar = open("selfies/" + selfie, 'rb')
twitter.update_profile_image(image=avatar)
def push(self, reqt):
return FifoDiskQueue.push(self, self.marshal(reqt))
def save(self, messages):
queue = FifoDiskQueue(self.database_name)
for message in messages:
queue.push(message)
queue.close()
def run(self, fn, sleep=1):
while True:
queue = FifoDiskQueue(self.database_name)
for message in self.generate_data(fn, sleep=sleep):
queue.push(message)
queue.close()
