def testBatch(self):
"""test saving, updating and deleting objects in batches"""
scores = [
GameScore(score=s, player_name='Jane', cheat_mode=False)
for s in range(5)
]
batcher = ParseBatcher()
batcher.batch_save(scores)
self.assertEqual(
GameScore.Query.filter(player_name='Jane').count(), 5,
"batch_save didn't create objects")
self.assertTrue(all(s.objectId is not None for s in scores),
"batch_save didn't record object IDs")
# test updating
for s in scores:
s.score += 10
batcher.batch_save(scores)
updated_scores = GameScore.Query.filter(player_name='Jane')
self.assertEqual(sorted([s.score for s in updated_scores]),
list(range(10, 15)),
msg="batch_save didn't update objects")
# test deletion
batcher.batch_delete(scores)
self.assertEqual(
GameScore.Query.filter(player_name='Jane').count(), 0,
"batch_delete didn't delete objects")
def main():
soup = BeautifulSoup(requests.get('https://www.mturk.com/mturk/viewhits?searchWords=&pageNumber=4&searchSpec=HITGroupSearch%23T%231%2310%23-1%23T%23%21%23%21NumHITs%211%21%23%21&sortType=NumHITs%3A1&selectedSearchType=hitgroups').text, "html.parser")
titles = soup.findAll('a', {"class" : "capsulelink"})
num_results = int(soup.findAll('td', {"class" : "title_orange_text"})[0].text.strip()[8:-7])
print("\nTotal number of HITs: " + str(num_results))
count = 0
page = 1
requestErrors = 0
privateCount = 0
register("DKJjvfvhnCGRK0cAdOpJN9MwR7zhIpuYya5xvbuF", "d8hIYrBrcW4r2ujEkL79vE03FmLxE2QCJgSwuXYv")
HITClass = ParseObject.factory("HIT")
all_hits = HITClass.Query.all()
batcher = ParseBatcher()
batcher.batch_delete(all_hits)
while (count < 200):
soup = BeautifulSoup(requests.get('https://www.mturk.com/mturk/viewhits?searchWords=&pageNumber=' + str(page) + '&searchSpec=HITGroupSearch%23T%231%2310%23-1%23T%23%21%23%21NumHITs%211%21%23%21&sortType=NumHITs%3A1&selectedSearchType=hitgroups').text, "html.parser")
titles = soup.findAll('a', {"class" : "capsulelink"})
for t in titles:
time.sleep(.3)
count = count + 1
print("\n" + str(count) + "\nTitle: " + t.text.strip())
linkA = t.parent.parent.findAll('span')[1].a
# check if the link is public
if linkA.has_attr('href'):
link = linkA['href']
hitPage = BeautifulSoup(requests.get('https://www.mturk.com' + link).text, "html.parser")
form = hitPage.findAll('form', {'name' : 'hitForm'})
# Check for error
if len(form) >= 3:
form = form[2]
requester = form.find("input", {'name' : 'prevRequester'})['value']
print('Requester: ' + requester)
reward = form.find("input", {'name' : 'prevReward'})['value']
print('Reward: ' + reward)
groupID = form.find("input", {'name' : 'groupId'})['value']
print('Group id: ' + groupID)
anyObject = HIT(requester=requester, reward=float(reward[3:]),
title=t.text.strip(), groupID=groupID)
anyObject.save()
else:
requestErrors = requestErrors + 1
print(link)
print(form)
else:
link = linkA['id']
print(link)
privateCount = privateCount + 1
page = page + 1
print("\n\nErrors: " + str(requestErrors))
print("Private HITs: " + str(privateCount))
def put_answers_in_Parse():
# Query for first 1000 Answers
queryset = list(Answer.Query.all().limit(1000))
while True:
if not queryset:
print("No Answers to delete from Parse -- none exist.")
break # skip to batch_save without deleting
elif len(queryset) == len(_Answer.LI_A):
print("{} Answers already exist in Parse.".format(len(queryset)))
srsly_delete_stuff = raw_input("Continue with delete anyway? (Y/n): ")
if srsly_delete_stuff != "Y":
print "Delete skipped. Upload skipped."
return
else:
print("There are {} Answers to delete from Parse.".format(len(queryset)))
srsly_delete_stuff = raw_input("Delete Answers from Parse? (Y/n): ")
if srsly_delete_stuff != "Y":
print "Delete skipped. Upload skipped."
return
# batch_delete in chunks of no more than 50
batcher = ParseBatcher()
lili_chunks = [queryset[i:i+50] for i in range(0, len(queryset), 50)]
for index, chunk in enumerate(lili_chunks):
batcher.batch_delete(chunk)
print "\r{} of {} Answers deleted from Parse".format(50*(index+1), len(queryset)),
sys.stdout.flush()
print
break # go to batch_save
# batch_save in chunks of no more than 50
len_lia = len(_Answer.LIA)
batcher = ParseBatcher()
lili_chunks = [_Answer.LIA[i:i+50] for i in range(0, len_lia, 50)]
for index, chunk in enumerate(lili_chunks):
while True:
try:
batcher.batch_save(chunk)
print "\r{} of {} Answers uploaded to Parse".format(50*(index+1), len_lia),
sys.stdout.flush()
break
except:
print("Locked. Sleeping for 5 seconds.")
time.sleep(5)
print
pass
def update_answers_in_Parse():
# Get a list of all Answers in Parse.
ct_a = Answer.Query.all().count()
queryset = []
batcher = ParseBatcher()
print("{} Answers exist in Parse.".format(ct_a))
if ct_a == 0: # None exist; upload whole list
pass
elif ct_a > 0: # There's at least 1 to get
for i in range(0, ct_a, min(ct_a,1000)): # for each chunk of <= 1000 answers
queryset += list(Answer.Query.all().skip(i).limit(1000)) # get the chunk, add to queryset
queryset.sort(key = attrgetter("num"))
for A, a in zip(queryset, [a for a in _Answer.LIA if queryset[a.num-1].num == a.num]): # for each answer with the same num
# compare all attributes of the _Answer class.
# if different, set Parse object's attribute to _Answer object's attribute;
# if all are same, keep in Parse and delete from LIA
for key in _Answer.LI_ATTR: # for all attributes of the _Answer class
if getattr(A, key) != getattr(a, key): # if different
print(key, getattr(A,key), getattr(a,key))
batcher.batch_delete([A])
batcher.batch_save([a])
# print("{} updated in Parse".format(a.ID))
break
elif _Answer.LI_ATTR[-1] == key:
_Answer.LIA.remove(a)
print("{} Answers updated in Parse.".format(len(queryset)-len(_Answer.LIA)))
print("{} Answers must be created in Parse.".format(len(_Answer.LIA)))
# Now, upload those remaining in _Answer.LIA to Parse
# (should put batch_upload_with_sleep in a separate function)
# batch_save in chunks of no more than 50
len_lia = len(_Answer.LIA)
batcher = ParseBatcher()
lili_chunks = [_Answer.LIA[i:i+50] for i in range(0, len_lia, 50)]
for index, chunk in enumerate(lili_chunks):
while True:
try:
batcher.batch_save(chunk)
print "\r{} of {} Answers uploaded to Parse".format(50*(index+1), len_lia),
sys.stdout.flush()
break
except:
print("Locked. Sleeping for 5 seconds.")
time.sleep(5)
print
def put_questions_in_Parse():
# Query for Questions
queryset = list(Question.Query.all().limit(1000))
while True:
if not queryset:
print("No Questions to delete from Parse -- none exist.")
break
elif len(queryset) == len(_Question.LI_Q):
print("{} Questions already exist in Parse.".format(len(queryset)))
srsly_delete_stuff = raw_input("Continue with delete anyway? (Y/n): ")
if srsly_delete_stuff != "Y":
print("Delete skipped. Upload skipped.")
return
else:
print("There are {} Questions to delete from Parse.".format(len(queryset)))
srsly_delete_stuff = raw_input("Delete Questions from Parse? (Y/n): ")
if srsly_delete_stuff != "Y":
print("Delete skipped. Upload skipped.")
return
# batch_delete in chunks of no more than 50
batcher = ParseBatcher()
lili_chunks = [queryset[i:i+50] for i in range(0, len(queryset), 50)]
for index, chunk in enumerate(lili_chunks):
batcher.batch_delete(chunk)
print("\r{} of {} Questions deleted from Parse".format(50*(index+1), len(queryset)), end = "\r")
sys.stdout.flush()
print
break
# batch_save in chunks of no more than 50
len_li_q = len(_Question.LIQ)
batcher = ParseBatcher()
lili_chunks = [_Question.LIQ[i:i+50] for i in range(0, len_li_q, 50)]
for index, chunk in enumerate(lili_chunks):
batcher.batch_save(chunk)
print("\r{} of {} Questions uploaded to Parse".format(50*(index+1), len_li_q), end = "\r")
sys.stdout.flush()
print
pass
def testBatch(self):
"""test saving, updating and deleting objects in batches"""
scores = [GameScore(score=s, player_name="Jane", cheat_mode=False) for s in range(5)]
batcher = ParseBatcher()
batcher.batch_save(scores)
self.assertEqual(GameScore.Query.filter(player_name="Jane").count(), 5, "batch_save didn't create objects")
self.assertTrue(all(s.objectId is not None for s in scores), "batch_save didn't record object IDs")
# test updating
for s in scores:
s.score += 10
batcher.batch_save(scores)
updated_scores = GameScore.Query.filter(player_name="Jane")
self.assertEqual(
sorted([s.score for s in updated_scores]), list(range(10, 15)), msg="batch_save didn't update objects"
)
# test deletion
batcher.batch_delete(scores)
self.assertEqual(GameScore.Query.filter(player_name="Jane").count(), 0, "batch_delete didn't delete objects")
class BatchDeleter:
def __init__(self, delay):
self.batcher = ParseBatcher()
self.objects_to_delete = []
self.delay = delay
self.delete_count = 0
def add_object_to_delete(self, object_to_delete):
self.objects_to_delete.append(object_to_delete)
if len(self.objects_to_delete) == 50:
self.delete()
def delete(self, delay=True):
if len(self.objects_to_delete) > 0:
print "Deleting", self.delete_count + 1, "-", self.delete_count + len(
self.objects_to_delete)
self.batcher.batch_delete(self.objects_to_delete)
self.delete_count += len(self.objects_to_delete)
self.objects_to_delete = []
if delay:
time.sleep(self.delay)
class ParseService(Base, LogMixin):
def __init__(self, settings):
self.settings = settings
self.fileRestClient = ParseFileRestClient(settings)
self.galleryService = GalleryService(settings)
register(self.settings["parse"]["application_id"], self.settings["parse"]["rest_api_key"])
self.batcher = ParseBatcher()
def getByFilePath(self, filePath):
return ContentItem.Query.get(filePath=filePath)
def post(self, item):
return item.save()
def drop(self):
# There is no truncate on parse, so we iterate and delete all...
if(self.settings["drop"]):
items = ContentItem.Query.all()
#self.logger.info(dir(items)
self.logger.info("Truncating items... %s" % items.count())
if items.count() > 0:
self.batcher.batch_delete(items)
self.logger.info("Done.")
class ParseService(Base, LogMixin):
def __init__(self, settings):
self.settings = settings
self.fileRestClient = ParseFileRestClient(settings)
self.galleryService = GalleryService(settings)
register(self.settings["parse"]["application_id"],
self.settings["parse"]["rest_api_key"])
self.batcher = ParseBatcher()
def getByFilePath(self, filePath):
return ContentItem.Query.get(filePath=filePath)
def post(self, item):
return item.save()
def drop(self):
# There is no truncate on parse, so we iterate and delete all...
if (self.settings["drop"]):
items = ContentItem.Query.all()
#self.logger.info(dir(items)
self.logger.info("Truncating items... %s" % items.count())
if items.count() > 0:
self.batcher.batch_delete(items)
self.logger.info("Done.")
screen.blit(label, (450, 68))
# back button
if button_alloc((450, 618, 85, 20)," remove last"):
enemies[selected_object].remove(enemies[selected_object][len(enemies[selected_object])-1])
# export
if button_alloc((450, 648, 85, 20)," export"):
# delete last one
dev_delete = pointBase.Query.filter(id_devices=device_ID)
dev_number = 1
while 1:
this_delete = dev_delete.limit(49)
if this_delete.exists():
batcher = ParseBatcher()
batcher.batch_delete(this_delete)
dev_delete.skip(49*dev_number).limit(49)
dev_number += 1
else:
break
for rocket_num, one in enumerate(enemies):
# create new one
rocket_id = str(time.time())
pointts = []
batcher = ParseBatcher()
save_limit = 0
all_object = 0
for count in range(0, len(one)):
if count == 0:
