def testBatch(self): """test saving, updating and deleting objects in batches""" scores = [ GameScore(score=s, player_name='Jane', cheat_mode=False) for s in range(5) ] batcher = ParseBatcher() batcher.batch_save(scores) self.assertEqual( GameScore.Query.filter(player_name='Jane').count(), 5, "batch_save didn't create objects") self.assertTrue(all(s.objectId is not None for s in scores), "batch_save didn't record object IDs") # test updating for s in scores: s.score += 10 batcher.batch_save(scores) updated_scores = GameScore.Query.filter(player_name='Jane') self.assertEqual(sorted([s.score for s in updated_scores]), list(range(10, 15)), msg="batch_save didn't update objects") # test deletion batcher.batch_delete(scores) self.assertEqual( GameScore.Query.filter(player_name='Jane').count(), 0, "batch_delete didn't delete objects")
def main(): soup = BeautifulSoup(requests.get('https://www.mturk.com/mturk/viewhits?searchWords=&pageNumber=4&searchSpec=HITGroupSearch%23T%231%2310%23-1%23T%23%21%23%21NumHITs%211%21%23%21&sortType=NumHITs%3A1&selectedSearchType=hitgroups').text, "html.parser") titles = soup.findAll('a', {"class" : "capsulelink"}) num_results = int(soup.findAll('td', {"class" : "title_orange_text"})[0].text.strip()[8:-7]) print("\nTotal number of HITs: " + str(num_results)) count = 0 page = 1 requestErrors = 0 privateCount = 0 register("DKJjvfvhnCGRK0cAdOpJN9MwR7zhIpuYya5xvbuF", "d8hIYrBrcW4r2ujEkL79vE03FmLxE2QCJgSwuXYv") HITClass = ParseObject.factory("HIT") all_hits = HITClass.Query.all() batcher = ParseBatcher() batcher.batch_delete(all_hits) while (count < 200): soup = BeautifulSoup(requests.get('https://www.mturk.com/mturk/viewhits?searchWords=&pageNumber=' + str(page) + '&searchSpec=HITGroupSearch%23T%231%2310%23-1%23T%23%21%23%21NumHITs%211%21%23%21&sortType=NumHITs%3A1&selectedSearchType=hitgroups').text, "html.parser") titles = soup.findAll('a', {"class" : "capsulelink"}) for t in titles: time.sleep(.3) count = count + 1 print("\n" + str(count) + "\nTitle: " + t.text.strip()) linkA = t.parent.parent.findAll('span')[1].a # check if the link is public if linkA.has_attr('href'): link = linkA['href'] hitPage = BeautifulSoup(requests.get('https://www.mturk.com' + link).text, "html.parser") form = hitPage.findAll('form', {'name' : 'hitForm'}) # Check for error if len(form) >= 3: form = form[2] requester = form.find("input", {'name' : 'prevRequester'})['value'] print('Requester: ' + requester) reward = form.find("input", {'name' : 'prevReward'})['value'] print('Reward: ' + reward) groupID = form.find("input", {'name' : 'groupId'})['value'] print('Group id: ' + groupID) anyObject = HIT(requester=requester, reward=float(reward[3:]), title=t.text.strip(), groupID=groupID) anyObject.save() else: requestErrors = requestErrors + 1 print(link) print(form) else: link = linkA['id'] print(link) privateCount = privateCount + 1 page = page + 1 print("\n\nErrors: " + str(requestErrors)) print("Private HITs: " + str(privateCount))
def put_answers_in_Parse(): # Query for first 1000 Answers queryset = list(Answer.Query.all().limit(1000)) while True: if not queryset: print("No Answers to delete from Parse -- none exist.") break # skip to batch_save without deleting elif len(queryset) == len(_Answer.LI_A): print("{} Answers already exist in Parse.".format(len(queryset))) srsly_delete_stuff = raw_input("Continue with delete anyway? (Y/n): ") if srsly_delete_stuff != "Y": print "Delete skipped. Upload skipped." return else: print("There are {} Answers to delete from Parse.".format(len(queryset))) srsly_delete_stuff = raw_input("Delete Answers from Parse? (Y/n): ") if srsly_delete_stuff != "Y": print "Delete skipped. Upload skipped." return # batch_delete in chunks of no more than 50 batcher = ParseBatcher() lili_chunks = [queryset[i:i+50] for i in range(0, len(queryset), 50)] for index, chunk in enumerate(lili_chunks): batcher.batch_delete(chunk) print "\r{} of {} Answers deleted from Parse".format(50*(index+1), len(queryset)), sys.stdout.flush() print break # go to batch_save # batch_save in chunks of no more than 50 len_lia = len(_Answer.LIA) batcher = ParseBatcher() lili_chunks = [_Answer.LIA[i:i+50] for i in range(0, len_lia, 50)] for index, chunk in enumerate(lili_chunks): while True: try: batcher.batch_save(chunk) print "\r{} of {} Answers uploaded to Parse".format(50*(index+1), len_lia), sys.stdout.flush() break except: print("Locked. Sleeping for 5 seconds.") time.sleep(5) print pass
def update_answers_in_Parse(): # Get a list of all Answers in Parse. ct_a = Answer.Query.all().count() queryset = [] batcher = ParseBatcher() print("{} Answers exist in Parse.".format(ct_a)) if ct_a == 0: # None exist; upload whole list pass elif ct_a > 0: # There's at least 1 to get for i in range(0, ct_a, min(ct_a,1000)): # for each chunk of <= 1000 answers queryset += list(Answer.Query.all().skip(i).limit(1000)) # get the chunk, add to queryset queryset.sort(key = attrgetter("num")) for A, a in zip(queryset, [a for a in _Answer.LIA if queryset[a.num-1].num == a.num]): # for each answer with the same num # compare all attributes of the _Answer class. # if different, set Parse object's attribute to _Answer object's attribute; # if all are same, keep in Parse and delete from LIA for key in _Answer.LI_ATTR: # for all attributes of the _Answer class if getattr(A, key) != getattr(a, key): # if different print(key, getattr(A,key), getattr(a,key)) batcher.batch_delete([A]) batcher.batch_save([a]) # print("{} updated in Parse".format(a.ID)) break elif _Answer.LI_ATTR[-1] == key: _Answer.LIA.remove(a) print("{} Answers updated in Parse.".format(len(queryset)-len(_Answer.LIA))) print("{} Answers must be created in Parse.".format(len(_Answer.LIA))) # Now, upload those remaining in _Answer.LIA to Parse # (should put batch_upload_with_sleep in a separate function) # batch_save in chunks of no more than 50 len_lia = len(_Answer.LIA) batcher = ParseBatcher() lili_chunks = [_Answer.LIA[i:i+50] for i in range(0, len_lia, 50)] for index, chunk in enumerate(lili_chunks): while True: try: batcher.batch_save(chunk) print "\r{} of {} Answers uploaded to Parse".format(50*(index+1), len_lia), sys.stdout.flush() break except: print("Locked. Sleeping for 5 seconds.") time.sleep(5) print
def put_questions_in_Parse(): # Query for Questions queryset = list(Question.Query.all().limit(1000)) while True: if not queryset: print("No Questions to delete from Parse -- none exist.") break elif len(queryset) == len(_Question.LI_Q): print("{} Questions already exist in Parse.".format(len(queryset))) srsly_delete_stuff = raw_input("Continue with delete anyway? (Y/n): ") if srsly_delete_stuff != "Y": print("Delete skipped. Upload skipped.") return else: print("There are {} Questions to delete from Parse.".format(len(queryset))) srsly_delete_stuff = raw_input("Delete Questions from Parse? (Y/n): ") if srsly_delete_stuff != "Y": print("Delete skipped. Upload skipped.") return # batch_delete in chunks of no more than 50 batcher = ParseBatcher() lili_chunks = [queryset[i:i+50] for i in range(0, len(queryset), 50)] for index, chunk in enumerate(lili_chunks): batcher.batch_delete(chunk) print("\r{} of {} Questions deleted from Parse".format(50*(index+1), len(queryset)), end = "\r") sys.stdout.flush() print break # batch_save in chunks of no more than 50 len_li_q = len(_Question.LIQ) batcher = ParseBatcher() lili_chunks = [_Question.LIQ[i:i+50] for i in range(0, len_li_q, 50)] for index, chunk in enumerate(lili_chunks): batcher.batch_save(chunk) print("\r{} of {} Questions uploaded to Parse".format(50*(index+1), len_li_q), end = "\r") sys.stdout.flush() print pass
def testBatch(self): """test saving, updating and deleting objects in batches""" scores = [GameScore(score=s, player_name="Jane", cheat_mode=False) for s in range(5)] batcher = ParseBatcher() batcher.batch_save(scores) self.assertEqual(GameScore.Query.filter(player_name="Jane").count(), 5, "batch_save didn't create objects") self.assertTrue(all(s.objectId is not None for s in scores), "batch_save didn't record object IDs") # test updating for s in scores: s.score += 10 batcher.batch_save(scores) updated_scores = GameScore.Query.filter(player_name="Jane") self.assertEqual( sorted([s.score for s in updated_scores]), list(range(10, 15)), msg="batch_save didn't update objects" ) # test deletion batcher.batch_delete(scores) self.assertEqual(GameScore.Query.filter(player_name="Jane").count(), 0, "batch_delete didn't delete objects")
class BatchDeleter: def __init__(self, delay): self.batcher = ParseBatcher() self.objects_to_delete = [] self.delay = delay self.delete_count = 0 def add_object_to_delete(self, object_to_delete): self.objects_to_delete.append(object_to_delete) if len(self.objects_to_delete) == 50: self.delete() def delete(self, delay=True): if len(self.objects_to_delete) > 0: print "Deleting", self.delete_count + 1, "-", self.delete_count + len( self.objects_to_delete) self.batcher.batch_delete(self.objects_to_delete) self.delete_count += len(self.objects_to_delete) self.objects_to_delete = [] if delay: time.sleep(self.delay)
class ParseService(Base, LogMixin): def __init__(self, settings): self.settings = settings self.fileRestClient = ParseFileRestClient(settings) self.galleryService = GalleryService(settings) register(self.settings["parse"]["application_id"], self.settings["parse"]["rest_api_key"]) self.batcher = ParseBatcher() def getByFilePath(self, filePath): return ContentItem.Query.get(filePath=filePath) def post(self, item): return item.save() def drop(self): # There is no truncate on parse, so we iterate and delete all... if(self.settings["drop"]): items = ContentItem.Query.all() #self.logger.info(dir(items) self.logger.info("Truncating items... %s" % items.count()) if items.count() > 0: self.batcher.batch_delete(items) self.logger.info("Done.")
class ParseService(Base, LogMixin): def __init__(self, settings): self.settings = settings self.fileRestClient = ParseFileRestClient(settings) self.galleryService = GalleryService(settings) register(self.settings["parse"]["application_id"], self.settings["parse"]["rest_api_key"]) self.batcher = ParseBatcher() def getByFilePath(self, filePath): return ContentItem.Query.get(filePath=filePath) def post(self, item): return item.save() def drop(self): # There is no truncate on parse, so we iterate and delete all... if (self.settings["drop"]): items = ContentItem.Query.all() #self.logger.info(dir(items) self.logger.info("Truncating items... %s" % items.count()) if items.count() > 0: self.batcher.batch_delete(items) self.logger.info("Done.")
screen.blit(label, (450, 68)) # back button if button_alloc((450, 618, 85, 20)," remove last"): enemies[selected_object].remove(enemies[selected_object][len(enemies[selected_object])-1]) # export if button_alloc((450, 648, 85, 20)," export"): # delete last one dev_delete = pointBase.Query.filter(id_devices=device_ID) dev_number = 1 while 1: this_delete = dev_delete.limit(49) if this_delete.exists(): batcher = ParseBatcher() batcher.batch_delete(this_delete) dev_delete.skip(49*dev_number).limit(49) dev_number += 1 else: break for rocket_num, one in enumerate(enemies): # create new one rocket_id = str(time.time()) pointts = [] batcher = ParseBatcher() save_limit = 0 all_object = 0 for count in range(0, len(one)): if count == 0: