def run(self): global global_l counter = 0 while self._stop_t == False: counter += 1 time.sleep(round(self.client.delay - (self.client.delay / 4.), 2)) if counter % self.client.batch_size == 0 and self.client.connection_error is False: print("sending data from thread {}:{}".format( self.client.cls_name(), id(self.client))) self.client.send_data(global_l[:self.client.batch_size]) global_l = global_l[self.client.batch_size:] counter = 0 if len(global_l) > 4 * self.client.batch_size: fq = FifoDiskQueue("{}.fifo.sql".format(self.client.name)) for obj in global_l[:self.client.batch_size * 2]: print("SAVED: {}".format(obj)) fq.push(obj) global_l = global_l[self.client.batch_size * 2:] fq.close() if (counter + 2) % self.client.batch_size == 0: fq = FifoDiskQueue("{}.fifo.sql".format(self.client.name)) if len(fq) > 0: for i in range(2): for elem in fq.pull(): print("PULL ELEM", elem) global_l.append(elem) print("DB SIZE", len(fq)) fq.close()
def gatherData(username): sequence = 0 VisitedNodes = set() queue = FifoDiskQueue("FriendsQueue") queue.push(username) sequence += 1 mapToRemoveDuplicates = {username: sequence} mapUsernameSequence = {username:sequence} anonymize(username, sequence) while (queue.__len__()>0): vertex = queue.pop() edges = parseFriends(vertex) VisitedNodes.add(vertex) count = VisitedNodes.__len__() if(count <= VertexLimit): for user in edges: if not VisitedNodes.__contains__(user): if (mapToRemoveDuplicates.get(user) == None): queue.push(user) sequence += 1 mapToRemoveDuplicates.update({user: sequence}) mapUsernameSequence.update({user:sequence}) anonymize(user, sequence) else: exit() print "Total Visited nodes " + str(count) saveData(vertex, edges) anonymizeDataset(vertex, edges, mapUsernameSequence)
def data_download(self): """ function for saving data from queue into PdfData table """ print("Starting data download") print("-" * 79) with self._app.app_context(): while 1: da_queue = FifoDiskQueue( os.path.dirname(__file__) + '/../queuedata') data = da_queue.pop() da_queue.close() if not data: print('sleep for 10') sleep(10) #break else: raw_data = json.loads(data.decode('utf-8')) info_log(self.logger.info, "Step1 Save into db Start", raw_data['reqd_data']) self._save_pdf_data(raw_data['reqd_data'], raw_data['tags'], raw_data['instance_id']) info_log(self.logger.info, "Step1 Save into db End", raw_data['reqd_data'])
def pop(queue): queue = FifoDiskQueue(queue) async_spec_bytes = queue.pop() queue.close() if async_spec_bytes: return json.loads(async_spec_bytes.decode()) else: return None
def run(self): queue = FifoDiskQueue(self.queue) while True: if self.query_cancel(): break qe = queue.pop() if qe == None: break task = qe.decode('latin1') os.system(task) queue.close()
def addSettings(): settings = request.form['settings'] if (settings != "null"): setting_queue = FifoDiskQueue("settings_file") setting_queue.push(settings.encode(encoding='UTF-8')) setting_queue.close() return "sucess" return "No settings to update"
def getTab(): tab = FifoDiskQueue("tab_file") content = [] t = tab.pop() while(t != None): t = t.decode(encoding='UTF-8') content.append(t) t = tab.pop() tab.close() return json.dumps(content)
def fetch_data(self): error = None tags = None try: if self.config['RANGE']: RANGE_NAME = self.config['SHEETNAME'] + "!" + self.config[ 'RANGE'] else: RANGE_NAME = self.config['SHEETNAME'] #call class method which return sheet data and error if permission is not there getValueMapping = self.get_sheetvalues(self.config['SHEETID'], RANGE_NAME) mappingError = getValueMapping[1] # Error in fetching mapping mappingValues = getValueMapping[0] # mapping values list if not mappingError: raw_data = mappingValues # Create a JSON from data. column_names = raw_data[0] final_data = [] for data in raw_data[2:]: single_data = dict() counter = 0 for col in column_names: single_data[col] = data[counter] counter += 1 tags = self.get_tags() all_data = dict() all_data['req_data'] = single_data all_data.update( self.config) # merge tags with sheet each row data #final_data.append(raw_data) raw_data = dict() raw_data['reqd_data'] = all_data raw_data['tags'] = tags q = FifoDiskQueue( os.path.dirname(__file__) + '/../../queuedata') q.push(json.dumps(raw_data).encode('utf-8')) q.close() else: error = "No Mapping details found" except Exception as e: import traceback error = "Failed to fetch mapping detials" mappingValues = None return error
def dataDownload(self): print("Starting data download") print("-" * 79) with self._app.app_context(): while 1: q = FifoDiskQueue(os.path.dirname(__file__) + '/../queuedata') data = q.pop() q.close() if not data: print('sleep for 10') sleep(10) #break else: raw_data = json.loads(data.decode('utf-8')) self.save_pdf_data(raw_data['reqd_data'], raw_data['tags'])
def update_all(root, image_label, cam, data_label): update_image(image_label, cam) interval = time.time() - start_time #print interval if(interval >2 ): saveimage(cam.read()) global q q = FifoDiskQueue("queuefile") #q.push(b global start_time start_time = time.time() #update_fps(fps_label) root.after(2, func=lambda: update_all(root, image_label, cam,data_label))
def addTab(): drink_name = request.form['drink_name'] if(drink_name != "null"): conn = getdbConn() tab = FifoDiskQueue("tab_file") drink_id = conn.execute('SELECT id FROM Drink WHERE name LIKE "' + drink_name + '" LIMIT 1;').fetchone() if(drink_id == None): return json.dumps({'success':False,'error':"NoDrinkInDB"}), 400, {'ContentType':'application/json'} tab.push(str(drink_id[0]).encode(encoding='UTF-8')) tab.close() conn.close() return json.dumps({'success':True, 'hello':'hey'}), 200, {'ContentType':'application/json'} return json.dumps({'success':False,'error':"DrinkNameNull"}), 400, {'ContentType':'application/json'}
def tap(): cabinet_id = request.args.get("cabinet_id") if(cabinet_id is not None): tab = FifoDiskQueue("tab_file") current_drink_id = tab.pop() if(current_drink_id != None): conn = getdbConn() drink_id = int(current_drink_id.decode(encoding='UTF-8')) tab.close() cursor = conn.execute(getSerachString(drink_id)) ingredients = [] for row in cursor: ingredients.append({ "name": row[1], "ratio": row[2] }) drink = { "drink_name": row[0], "ingredients": ingredients } response = {"settings":"null","drink":drink} return json.dumps(response) return json.dumps({"settings": "null", "drink":"null"}) else: return json.dumps({'success': False,'error':"Must Have Cabinet ID to access to tap"}), 400, {'ContentType':'application/json'}
def push(queue, async_spec): queue = FifoDiskQueue(queue) queue.push(json.dumps(async_spec).encode()) queue.close()
# use lock to visited links so only one thread can update at a time visited_lock = threading.Lock() # make visited links a hashed set so there are not duplicates # a bloom filter may improve performance with less memory visited_links = set() # create a queue of unvisited links added by threads as they scrape if os.path.isdir(BUFFER_FILE): shutil.rmtree(BUFFER_FILE) elif os.path.isfile(BUFFER_FILE): os.remove(BUFFER_FILE) if int(search_type) == 1: # BFS / DIR disk_buffer = FifoDiskQueue(BUFFER_FILE) else: # DFS / FILE disk_buffer = LifoDiskQueue(BUFFER_FILE) unvisited_links_in = Queue() unvisited_links_out = Queue() unparsed_html = JoinableQueue() buffer_lock = threading.Lock() first_link = dict() first_link['url'] = start_url first_link['parent_url'] = None first_link['level'] = 0 # add first link to queue disk_buffer.push(pickle.dumps(first_link))
def add_job(self, s): queue = FifoDiskQueue(self.queue) queue.push(s.encode('latin1')) queue.close()
from queuelib import FifoDiskQueue from apscheduler.schedulers.blocking import BlockingScheduler import api_keys sched = BlockingScheduler() APP_KEY = api_keys.APP_KEY APP_SECRET = api_keys.APP_SECRET OAUTH_TOKEN = api_keys.OAUTH_TOKEN OAUTH_TOKEN_SECRET = api_keys.OAUTH_TOKEN_SECRET twitter = Twython(APP_KEY, APP_SECRET, OAUTH_TOKEN, OAUTH_TOKEN_SECRET) selfies = ['1.png', '2.png', '3.png'] q = FifoDiskQueue("queuefile") if len(q) == 0: for selfie in selfies: q.push(selfie) selfie = q.pop() q.close() @sched.scheduled_job('interval', minutes=10) def revolve_avatar(): avatar = open("selfies/" + selfie, 'rb') twitter.update_profile_image(image=avatar)
def initializeQueue(self): """initialize the queue instance.""" queuedir = self.log_dir + '/b2safe.queue' self.queue = FifoDiskQueue(queuedir)
def fetch_data(self): error = None tags = None try: reqData = json.loads(json.dumps(request.json)) # Converting unicodes to str and then str to dict. formId = reqData[ 'formId'] # Getting the form ID to distinguish between various template documents and mapping sheets newReqData = reqData['data'][0] # Getting the data : [{values}] instanceId = newReqData['instanceID'] # Getting the instance id for searching routes newReqData = json.loads(json.dumps(newReqData)) # Getting the new data userName = newReqData['username'] formSubmissionDate = newReqData[ '*meta-submission-date*'] # Correcting the submission date and removing the time endIndex = formSubmissionDate.find(str('T')) formSubmissionDate = formSubmissionDate[:endIndex] newReqData['*meta-submission-date*'] = formSubmissionDate # Saving the corrected date in the json myDict = {} for k, v in newReqData.items(): if type(v) is dict: for k1, v1 in v.items(): if k1 == "url": # correcting the URLs baseUrl = 'http://aggregate.cttsamagra.xyz:8080/' indexStart = 0 # Finding the substring indexEnd = v1.find( ":8080/") + 6 # Find the stopping point newv1 = v1.replace(v1[indexStart:indexEnd], baseUrl) myDict[k] = newv1 elif type(v) is float or type(v) is int: myDict[k] = str(v) elif type(v) is list: myDict[k] = str(v[0]) # Converting list to str else: if v is None: v = "NO_TEXT_FOUND" myDict[k] = v # Calculate Udise from its database and then Calculate distance from udise calculated_distance = 'Not available' # Calculate using udise myDict['calculated_distance'] = calculated_distance all_data = dict() all_data['req_data'] = myDict all_data['FORMID'] = formId all_data['INSTANCEID'] = instanceId all_data['USERNAME'] = userName all_data['FORMSUBMISSIONDATE'] = formSubmissionDate self.raw_data = all_data tags = self.get_tags() all_data.update(self.config) raw_data = dict() raw_data['reqd_data'] = all_data raw_data['tags'] = tags q = FifoDiskQueue(os.path.dirname(__file__)+'/../../queuedata') q.push(json.dumps(raw_data).encode('utf-8')) q.close() except Exception as e: import traceback error = "Failed to fetch mapping detials" return error
def __init__(self): super(Thingsspeak, self).__init__() self.retry_queue = FifoDiskQueue("/var/thingsspeak.queue")