def remove_item(scan_path):
try:
return DeleteQuery(QueueItemModel).where(
QueueItemModel.scan_path == scan_path).execute()
except Exception:
logger.exception("Exception deleting %r from database: ", scan_path)
return False
def test_delete(self):
InsertQuery(Blog, title='a').execute()
InsertQuery(Blog, title='b').execute()
InsertQuery(Blog, title='c').execute()
dq = DeleteQuery(Blog).where(title='b')
self.assertEqual(dq.sql(), ('DELETE FROM blog WHERE title = ?', ['b']))
self.assertEqual(dq.execute(), 1)
sq = SelectQuery(Blog).order_by('id')
self.assertEqual([x.title for x in sq], ['a', 'c'])
dq = DeleteQuery(Blog)
self.assertEqual(dq.sql(), ('DELETE FROM blog', []))
self.assertEqual(dq.execute(), 2)
sq = SelectQuery(Blog).order_by('id')
self.assertEqual([x.title for x in sq], [])
def purge_invalid_proxies():
# Set a limit of 15 to ensure we don't grab more than we can handle.
limit = 15
urls = []
try:
query = (ProxyPool
.select()
.where(ProxyPool.invalid == 1)
.limit(limit))
for p in query:
urls.append(p.url)
if len(urls) > 0:
log.info('Retrieved proxies for deletion: {}'.format(urls))
result = DeleteQuery(ProxyPool).where(
ProxyPool.url << urls).execute()
log.info('Deleted {} invalid proxies.'.format(result))
except OperationalError as e:
log.error('Failed purge invalid proxies query: {}'.format(e))
def add_hooks_to_db(self, job: Job):
# First delete all hooks to avoid spending too much time
# updating, deleting, etc ...
try:
DeleteQuery(Hook).where(Hook.job == job.id).execute()
except OperationalError as e:
pass
hooks_added = list()
for hook, params in self._hooks.items():
hook = Hook(name=hook,
when=params['when'],
method=params['method'],
priority=params['priority'],
description=params['description'],
active=params['active'],
job=job)
hook.save()
hooks_added.append(hook.id)
return hooks_added
def test_delete(self):
InsertQuery(Blog, title="a").execute()
InsertQuery(Blog, title="b").execute()
InsertQuery(Blog, title="c").execute()
dq = DeleteQuery(Blog).where(title="b")
self.assertEqual(dq.sql(), ("DELETE FROM blog WHERE title = ?", ["b"]))
self.assertEqual(dq.execute(), 1)
sq = SelectQuery(Blog).order_by("id")
self.assertEqual([x.title for x in sq], ["a", "c"])
dq = DeleteQuery(Blog)
self.assertEqual(dq.sql(), ("DELETE FROM blog", []))
self.assertEqual(dq.execute(), 2)
sq = SelectQuery(Blog).order_by("id")
self.assertEqual([x.title for x in sq], [])
def test_delete(self):
InsertQuery(Blog, title='a').execute()
InsertQuery(Blog, title='b').execute()
InsertQuery(Blog, title='c').execute()
dq = DeleteQuery(Blog).where(title='b')
self.assertEqual(dq.sql(), ('DELETE FROM blog WHERE title = ?', ['b']))
self.assertEqual(dq.execute(), 1)
sq = SelectQuery(Blog).order_by('id')
self.assertEqual([x.title for x in sq], ['a', 'c'])
dq = DeleteQuery(Blog)
self.assertEqual(dq.sql(), ('DELETE FROM blog', []))
self.assertEqual(dq.execute(), 2)
sq = SelectQuery(Blog).order_by('id')
self.assertEqual([x.title for x in sq], [])
def parse_gyms(args, gym_responses, wh_update_queue):
gym_details = {}
gym_members = {}
gym_pokemon = {}
trainers = {}
i = 0
for g in gym_responses.values():
gym_state = g['gym_state']
gym_id = gym_state['fort_data']['id']
gym_details[gym_id] = {
'gym_id': gym_id,
'name': g['name'],
'description': g.get('description'),
'url': g['urls'][0],
}
if args.webhooks:
webhook_data = {
'id': gym_id,
'latitude': gym_state['fort_data']['latitude'],
'longitude': gym_state['fort_data']['longitude'],
'team': gym_state['fort_data'].get('owned_by_team', 0),
'name': g['name'],
'description': g.get('description'),
'url': g['urls'][0],
'pokemon': [],
}
for member in gym_state.get('memberships', []):
gym_members[i] = {
'gym_id': gym_id,
'pokemon_uid': member['pokemon_data']['id'],
}
gym_pokemon[i] = {
'pokemon_uid':
member['pokemon_data']['id'],
'pokemon_id':
member['pokemon_data']['pokemon_id'],
'cp':
member['pokemon_data']['cp'],
'trainer_name':
member['trainer_public_profile']['name'],
'num_upgrades':
member['pokemon_data'].get('num_upgrades', 0),
'move_1':
member['pokemon_data'].get('move_1'),
'move_2':
member['pokemon_data'].get('move_2'),
'height':
member['pokemon_data'].get('height_m'),
'weight':
member['pokemon_data'].get('weight_kg'),
'stamina':
member['pokemon_data'].get('stamina'),
'stamina_max':
member['pokemon_data'].get('stamina_max'),
'cp_multiplier':
member['pokemon_data'].get('cp_multiplier'),
'additional_cp_multiplier':
member['pokemon_data'].get('additional_cp_multiplier', 0),
'iv_defense':
member['pokemon_data'].get('individual_defense', 0),
'iv_stamina':
member['pokemon_data'].get('individual_stamina', 0),
'iv_attack':
member['pokemon_data'].get('individual_attack', 0),
'last_seen':
datetime.utcnow(),
}
trainers[i] = {
'name': member['trainer_public_profile']['name'],
'team': gym_state['fort_data']['owned_by_team'],
'level': member['trainer_public_profile']['level'],
'last_seen': datetime.utcnow(),
}
if args.webhooks:
webhook_data['pokemon'].append({
'pokemon_uid':
member['pokemon_data']['id'],
'pokemon_id':
member['pokemon_data']['pokemon_id'],
'cp':
member['pokemon_data']['cp'],
'num_upgrades':
member['pokemon_data'].get('num_upgrades', 0),
'move_1':
member['pokemon_data'].get('move_1'),
'move_2':
member['pokemon_data'].get('move_2'),
'height':
member['pokemon_data'].get('height_m'),
'weight':
member['pokemon_data'].get('weight_kg'),
'stamina':
member['pokemon_data'].get('stamina'),
'stamina_max':
member['pokemon_data'].get('stamina_max'),
'cp_multiplier':
member['pokemon_data'].get('cp_multiplier'),
'additional_cp_multiplier':
member['pokemon_data'].get('additional_cp_multiplier', 0),
'iv_defense':
member['pokemon_data'].get('individual_defense', 0),
'iv_stamina':
member['pokemon_data'].get('individual_stamina', 0),
'iv_attack':
member['pokemon_data'].get('individual_attack', 0),
'trainer_name':
member['trainer_public_profile']['name'],
'trainer_level':
member['trainer_public_profile']['level'],
})
i += 1
if args.webhooks:
wh_update_queue.put(('gym_details', webhook_data))
# All this database stuff is synchronous (not using the upsert queue) on purpose.
# Since the search workers load the GymDetails model from the database to determine if a gym
# needs rescanned, we need to be sure the GymDetails get fully committed to the database before moving on.
#
# We _could_ synchronously upsert GymDetails, then queue the other tables for
# upsert, but that would put that Gym's overall information in a weird non-atomic state.
# upsert all the models
if len(gym_details):
bulk_upsert(GymDetails, gym_details)
if len(gym_pokemon):
bulk_upsert(GymPokemon, gym_pokemon)
if len(trainers):
bulk_upsert(Trainer, trainers)
# get rid of all the gym members, we're going to insert new records
if len(gym_details):
DeleteQuery(GymMember).where(
GymMember.gym_id << gym_details.keys()).execute()
# insert new gym members
if len(gym_members):
bulk_upsert(GymMember, gym_members)
log.info('Upserted %d gyms and %d gym members', len(gym_details),
len(gym_members))
def __init__(self, *args, **kwargs):
BaseDeleteQuery.__init__(self, *args, **kwargs)
gen.Future.__init__(self)
self._future = None
def process_gym_details(self, json_data):
self.gym_details += 1
if args.no_gymdetail:
return
monkey = False
if 'gym_defenders' in json_data:
monkey = True
to_keep = ["gym_id", "name", "description", "url"]
gymdetails = {}
if monkey is False:
id = json_data['id']
gymdetails[id] = json_data
# copy this for webhook forwarding
wh_gymdetails = gymdetails[id].copy()
# the wh sends "id", the but the database
# wants gym_id
gymdetails[id].update({'gym_id': gymdetails[id]['id']})
if monkey is True:
id = json_data['gym_id']
gymdetails[id] = json_data
wh_gymdetails = gymdetails[id].copy()
gymdetails[id].update({
'gym_id': id,
'name': gymdetails[id]['name'],
'description': gymdetails[id]['name'],
'url': gymdetails[id]['url']
})
# I found a gym that doesn't send a name, so we'll put this here.
if gymdetails[id]['name'] is None:
gymdetails[id].update({'name': "None"})
# DB needs something not NULL
if json_data['url'] is None:
gymdetails[id]['url'] = "http://notgiven.com"
# we need to extract trainer and pokemon information before
# getting gymdetails ready for the database
(gym_pokemon, gym_members,
trainers) = self.process_gympokemon(id, monkey, gymdetails)
# copies all the keys we want for the DB
gymdetails[id] = {
key: gymdetails[id][key]
for key in gymdetails[id] if key in to_keep
}
log.debug("%s", gymdetails)
# put it all into the db queue
db_queue.put((GymDetails, gymdetails))
db_queue.put((Trainer, trainers))
db_queue.put((GymPokemon, gym_pokemon))
# Need to delete gym members
DeleteQuery(GymMember).where(
GymMember.gym_id << gymdetails.keys()).execute()
db_queue.put((GymMember, gym_members))
if args.webhooks:
wh_queue.put(('gym_details', wh_gymdetails))
def clear():
query = DeleteQuery(DbJiraIssues)
query.execute()
