def __init__(self):
self.config = Config()
self.path_graph = self.config.get("sphinx", "AutoDocSchemaDisplay", "path_graph", "")
# Инициализация параметров логирования.
logging.basicConfig(level=int(self.config.get("logging", "console", "level", "10")), stream=sys.stdout,
format='%(asctime)s [%(module)15s] [%(funcName)19s] [%(lineno)4d] [%(levelname)7s] [%(threadName)4s] %(message)s')
def __init__(self):
self.config = Config()
self.position_latitude = self.config_xml.get("AI", "location",
"latitude", "55.0467")
self.position_longitude = self.config_xml.get("AI", "location",
"longitude", "55.0467")
self.position = [self.position_latitude, self.position_longitude]
self.api = PGoApi()
def __init__(self, import_name, **kwargs):
super(Map, self).__init__(import_name, **kwargs)
self.config_xml = Config()
self.json_encoder = CustomJSONEncoder
self.route("/", methods=['GET'])(self.index)
self._flask_json()
self._flask_report()
self.conf_latitude = self.config_xml.get("map", "", "latitude", "55.0467")
self.conf_longitude = self.config_xml.get("map", "", "longitude", "73.3111")
self.conf_gmapkey = self.config_xml.get("map", "google", "key", "")
class AI:
def __init__(self):
self.config = Config()
self.position_latitude = self.config_xml.get("AI", "location",
"latitude", "55.0467")
self.position_longitude = self.config_xml.get("AI", "location",
"longitude", "55.0467")
self.position = [self.position_latitude, self.position_longitude]
self.api = PGoApi()
def _login(self):
self.account_service = self.config.get("AI", "config", "service",
"ptc")
self.account_username = self.config.get("AI", "config", "username", "")
self.account_password = self.config.get("AI", "config", "password", "")
self.api.set_position(*self.position)
self.api.login(self.account_service, self.account_username,
self.account_password)
def run(self):
self.alive = True
self.config = Config()
self.geolocation = Geolocation(self)
self.session_maker = init(self.config)
self.session = self.session_maker()
self.scanner = self.session.query(dbScanner).get(self.scanner_id)
self.metrica = Metrica(self)
self.api = PGoApi(metrica=self.metrica)
self.api.activate_signature(self.config.get("AI", "", "signature", "tess"))
if self.scanner.proxy:
log.info("Используем Proxy!")
self.api.set_proxy({
"https": "http://{}:{}".format(self.scanner.proxy.ip, self.scanner.proxy.port)
})
self.profile = Profile(self)
self.inventory = Inventory(self)
self.ai = AI(self)
self.metrica.take_status(scanner_state=1, scanner_msg="Запускаем сканнер", account_state=1)
log.info("Запускаем сканнер")
while not self._stopevent.isSet():
if not self.run_scanner():
self._stopevent.set()
break
else:
self.metrica.take_status(scanner_state=1, scanner_msg="Ожидаем следующего цикла")
log.info("Ожидаем следующего цикла")
self._stopevent.wait(randint(self._sleepperiod, self._sleepperiod*2))
try:
self.metrica.take_status(scanner_state=0, scanner_msg="Сканнер отключен")
self.session.commit()
self.session.flush()
finally:
self.session.close()
self.alive = False
def __init__(self):
self.config = Config()
self.queue = Queue()
self.cache_path = self.config.get("yator", "cache", "path", "")
self.structure_path = self.config.get("yator", "structure", "path", "")
self.bitrate = self.config.get("transcode", "", "bitrate", "256")
self.library = Library(self.config.get("itunes", "", "library", ""))
self.playlist_re = re.compile("Yator\sCD(?P<disk>\d\d)")
self.structure_prepare()
self.queue_prepare()
self.queue_worker()
self.song_rebase()
def __init__(self, scanner_id):
threading.Thread.__init__(self, name=scanner_id)
self.config = Config()
self.geolocation = Geolocation(self)
self.session_maker = init(self.config)
self.session = self.session_maker()
self.scanner = self.session.query(dbScanner).get(scanner_id)
self._stopevent = threading.Event()
self._sleepperiod = 10
self._sleeplogin = 20
self.api = PGoApi()
self.profile = Profile(self)
self.inventory = Inventory(self)
self.metrica = Metrica(self)
self.ai = AI(self)
self.daemon = True
class Scanner(threading.Thread):
def __init__(self, scanner_id):
threading.Thread.__init__(self, name=scanner_id)
self.scanner_id = scanner_id
self.config = None
self.geolocation = None
self.session_maker = None
self.session = None
self.scanner = None
self._stopevent = threading.Event()
self._sleepperiod = 10
self._sleeplogin = 20
self.api = None
self.profile = None
self.inventory = None
self.metrica = None
self.ai = None
self.daemon = True
def _scanner_snapshot(self, scanner_id):
scanner = self.session.query(dbScanner).get(scanner_id)
scanner.location.fix(self.geolocation)
return {
"location": {
"latitude": scanner.location.latitude,
"longitude": scanner.location.longitude,
"position": [scanner.location.latitude, scanner.location.longitude, 0],
"distance": scanner.location.distance
},
"account": {
"username": scanner.account.username,
"password": scanner.account.password,
"service": scanner.account.service
},
"mode": {
"walk": scanner.mode.walk,
"step": scanner.mode.step,
"stepper": scanner.mode.stepper,
"is_catch": scanner.mode.is_catch,
"is_farm": scanner.mode.is_farm,
"is_defender": scanner.mode.is_defender,
"is_lookup": scanner.mode.is_lookup,
"is_search": scanner.mode.is_search
}
}
def login(self):
login_count = 0
login_count_max = 5
self.metrica.take_status(account_state=1, account_msg="Попытка авторизации ({0})".format(login_count))
log.debug("Попытка авторизации ({0})".format(login_count))
if self.scanner.latitude != 0 and self.scanner.longitude != 0:
self.api.set_position(self.scanner.latitude, self.scanner.longitude, 0)
else:
self.api.set_position(self.scanner.location.latitude, self.scanner.location.longitude, 0)
while not self.api.login(self.scanner.account.service, self.scanner.account.username, self.scanner.account.password):
if login_count < login_count_max:
self.metrica.take_status(account_state=1, account_msg="Ошибка авторизации ({0}), ожидаем".format(login_count))
log.warning("Ошибка авторизации ({0}), ожидаем".format(login_count))
self._stopevent.wait(randint(self._sleeplogin, self._sleeplogin*3))
login_count += 1
else:
self.metrica.take_status(account_state=0, account_msg="Успешная авторизация")
log.warning("Ошибка авторизации, выходим")
return False
self.metrica.take_status(account_state=1, account_msg="Успешная авторизация")
log.info("Успешная авторизация")
return True
def run(self):
self.alive = True
self.config = Config()
self.geolocation = Geolocation(self)
self.session_maker = init(self.config)
self.session = self.session_maker()
self.scanner = self.session.query(dbScanner).get(self.scanner_id)
self.metrica = Metrica(self)
self.api = PGoApi(metrica=self.metrica)
self.api.activate_signature(self.config.get("AI", "", "signature", "tess"))
if self.scanner.proxy:
log.info("Используем Proxy!")
self.api.set_proxy({
"https": "http://{}:{}".format(self.scanner.proxy.ip, self.scanner.proxy.port)
})
self.profile = Profile(self)
self.inventory = Inventory(self)
self.ai = AI(self)
self.metrica.take_status(scanner_state=1, scanner_msg="Запускаем сканнер", account_state=1)
log.info("Запускаем сканнер")
while not self._stopevent.isSet():
if not self.run_scanner():
self._stopevent.set()
break
else:
self.metrica.take_status(scanner_state=1, scanner_msg="Ожидаем следующего цикла")
log.info("Ожидаем следующего цикла")
self._stopevent.wait(randint(self._sleepperiod, self._sleepperiod*2))
try:
self.metrica.take_status(scanner_state=0, scanner_msg="Сканнер отключен")
self.session.commit()
self.session.flush()
finally:
self.session.close()
self.alive = False
def run_scanner(self):
try:
if self.api._auth_provider and self.api._auth_provider._ticket_expire:
remaining_time = self.api._auth_provider._ticket_expire/1000 - time.time()
if remaining_time > 60:
self.metrica.take_status(scanner_state=1, scanner_msg="Уже залогинены, прускаем")
log.debug("Уже залогинены, прускаем")
else:
if not self.login():
self.metrica.take_status(scanner_state=1, scanner_msg="Ошибка авторизации, выходим")
log.warning("Ошибка авторизации, выходим")
return False
else:
if not self.login():
self.metrica.take_status(scanner_state=1, scanner_msg="Ошибка авторизации, выходим")
log.warning("Ошибка авторизации, выходим")
return False
except Exception as e:
self.metrica.take_status(scanner_state=1, scanner_msg="Ошибка авторизации, выходим")
log.error("Ошибка авторизации, выходим")
log.error(str(e))
return False
try:
self.profile.update()
finally:
self.session.flush()
log.info("Начинаем обработку")
self.ai.take_step()
#self.ai.heartbeat()
return True
def join(self, timeout=None):
log.info("Был получен сигнал заверщения работы")
self.metrica.take_status(scanner_state=1, scanner_msg="Был получен сигнал заверщения работы")
self._stopevent.set()
self.ai.search.stop()
threading.Thread.join(self, timeout)
try:
self.metrica.take_status(scanner_state=0, scanner_msg="Сканнер отключен")
self.session.commit()
self.session.flush()
finally:
self.session.close()
log.info("Поток завершил работу")
class Yator:
def __init__(self):
self.config = Config()
self.queue = Queue()
self.cache_path = self.config.get("yator", "cache", "path", "")
self.structure_path = self.config.get("yator", "structure", "path", "")
self.bitrate = self.config.get("transcode", "", "bitrate", "256")
self.library = Library(self.config.get("itunes", "", "library", ""))
self.playlist_re = re.compile("Yator\sCD(?P<disk>\d\d)")
self.structure_prepare()
self.queue_prepare()
self.queue_worker()
self.song_rebase()
def structure_prepare(self):
if not os.path.isdir(self.cache_path):
os.makedirs(self.cache_path, 0755)
if not os.path.isdir(self.structure_path):
os.makedirs(self.structure_path, 0755)
# Делаем минимальную структуру для работы сд ченжера
#
structure_minimal_cd = self.config.get("yator", "structure", "minimal_cd", "6")
for cd_index in range(1, int(structure_minimal_cd), 1):
if len(str(cd_index)) == 1:
cd_index = "0{0}".format(cd_index)
cd_path = "{0}CD{1}".format(self.structure_path, str(cd_index))
if not os.path.isdir(cd_path):
os.makedirs(cd_path, 0755)
def song_rebase(self):
playlists = self.library.getPlaylistNames()
for playlist in playlists:
playlist_match = self.playlist_re.search(playlist)
if playlist_match:
disk_number = playlist_match.group('disk')
disk_folder = "{0}CD{1}/".format(self.structure_path, disk_number)
print "Заполняем структуру по плейлисту \'{0}\' как диск {1} с {2} композицией(ями)".format(playlist,
disk_number, len(self.library.getPlaylist(playlist).tracks))
if not os.path.isdir(disk_folder):
os.makedirs(disk_folder, 0755)
else:
filelist = [f for f in os.listdir(disk_folder) if f.endswith(".mp3")]
for f in filelist:
os.remove("{0}/{1}".format(disk_folder, f))
for song in self.library.getPlaylist(playlist).tracks:
print " Перемещаем #{2} [{0} - {1}]".format(song.artist, song.name, song.playlist_order)
file_cache = "{0}{1}.mp3".format(self.cache_path, song.track_id)
file_name = self.make_safe_filename(
translit("{0}_{1} - {2}.mp3".format(song.playlist_order, song.artist,
song.name),
language_code="ru", reversed=True))
file_structure = "{0}{1}".format(disk_folder, file_name)
shutil.copyfile(file_cache, file_structure)
def queue_prepare(self):
print 'Подготавливаем очередь для транскодинга...'
self.library = Library(self.config.get("itunes", "", "library", ""))
playlists = self.library.getPlaylistNames()
for playlist in playlists:
playlist_match = self.playlist_re.search(playlist)
if playlist_match:
print "Нашли плейлист \'{0}\' с {1} композицией(ями)".format(playlist,
len(self.library.getPlaylist(playlist).tracks))
for song in self.library.getPlaylist(playlist).tracks:
self.queue.put(["/{0}".format(song.location), "{0}{1}.mp3".format(self.cache_path, song.track_id)])
def queue_worker(self):
thread_count = self.config.get("transcode", "", "threads", "2")
print 'Зупускаем {0} транскодеров(а) c битрейтом в {1} килобит в секунду.'.format(thread_count, self.bitrate)
for thread_index in xrange(0, int(thread_count), 1):
worker = Thread(target=self.thread_worker, args=(thread_index, self.queue, self.bitrate))
worker.setDaemon(True)
worker.start()
self.queue.join()
print 'Транскодинг завершен.'
def thread_worker(self, thread_index, queue, file_bitrate):
transcode = AudioTranscode()
while True:
[file_orig, file_transcoded] = queue.get()
print '{0}: {1}'.format(thread_index, file_orig)
transcode.transcode(file_orig, file_transcoded, bitrate=file_bitrate)
queue.task_done()
@staticmethod
def make_safe_filename(filename):
try:
safechars = string.letters + string.digits + " -_."
return filter(lambda c: c in safechars, filename)
except:
return ""
pass
class Map(Flask):
def __init__(self, import_name, **kwargs):
super(Map, self).__init__(import_name, **kwargs)
self.config_xml = Config()
self.json_encoder = CustomJSONEncoder
self.route("/", methods=['GET'])(self.index)
self._flask_json()
self._flask_report()
self.conf_latitude = self.config_xml.get("map", "", "latitude", "55.0467")
self.conf_longitude = self.config_xml.get("map", "", "longitude", "73.3111")
self.conf_gmapkey = self.config_xml.get("map", "google", "key", "")
def _flask_report(self):
# trainers
self.add_url_rule("/report/trainer/top", view_func=ReportTrainerTop.as_view("report/trainer/top", config=self.config_xml))
# Gyms
self.add_url_rule("/report/gym/membership", view_func=ReportGymMembership.as_view("report/gym/membership", config=self.config_xml))
self.add_url_rule("/report/gym/top", view_func=ReportGymTop.as_view("report/gym/top", config=self.config_xml))
self.add_url_rule("/report/gym/team", view_func=ReportGymTeam.as_view("report/gym/team", config=self.config_xml))
self.add_url_rule("/report/gym/guard", view_func=ReportGymGuard.as_view("report/gym/guard", config=self.config_xml))
# Pokemon
self.add_url_rule("/report/pokemon/average", view_func=ReportPokemonAverage.as_view("report/pokemon/average", config=self.config_xml))
self.add_url_rule("/report/pokemon/now", view_func=ReportPokemonNow.as_view("report/pokemon/now", config=self.config_xml))
# Servers
self.add_url_rule("/report/server/average", view_func=ReportServerAverage.as_view("report/server/average", config=self.config_xml))
self.add_url_rule("/report/server/account", view_func=ReportServerAccount.as_view("report/server/account", config=self.config_xml))
def _flask_json(self):
self.route("/put_user_geo", methods=['GET'])(self.json_put_user_geo)
self.route("/get_data", methods=['GET'])(self.json_get_data)
self.route("/put_data_geo", methods=['GET'])(self.json_put_data_geo)
def _database_init(self):
session_maker = init(self.config_xml)
session = session_maker()
return session
def _database_close(self, session):
session.flush()
session.expunge_all()
session.close()
def _database_fetch_pokemons(self, session=None, ne_latitude=0, ne_longitude=0, sw_latitude=0, sw_longitude=0):
pokemons = []
sql = """
SELECT
p.id as "id",
p.name as "name",
ps.cd_encounter as "encounter_id",
ps.latitude as "latitude",
ps.longitude as "longitude",
ps.date_disappear as "date_disappear"
FROM
db_pokestats.pokemon as p,
db_pokestats.pokemon_spawnpoint ps
WHERE
ps.latitude < {0}
and ps.latitude > {1}
and ps.longitude < {2}
and ps.longitude > {3}
and ps.date_disappear > now()
and ps.cd_pokemon = p.id
""".format(ne_latitude, sw_latitude, ne_longitude, sw_longitude)
for row in session.execute(sqlalchemy.text(sql)):
pokemons.append({
"pokemon_id": row[0],
"pokemon_name": row[1],
"encounter_id": row[2],
"latitude": row[3],
"longitude": row[4],
"date_disappear": row[5]
})
return pokemons
def _database_fetch_pokestops(self, session=None, ne_latitude=0, ne_longitude=0, sw_latitude=0, sw_longitude=0):
pokestops = []
sql = """
SELECT
p.id as "id",
p.name as "name",
p.image_url as "image",
p.latitude as "latitude",
p.longitude as "longitude",
p.date_lure_expiration as "date_lure_expiration",
p.date_change as "date_change"
FROM
db_pokestats.pokestop as p
WHERE
p.latitude < {0}
and p.latitude > {1}
and p.longitude < {2}
and p.longitude > {3}
""".format(ne_latitude, sw_latitude, ne_longitude, sw_longitude)
for row in session.execute(sqlalchemy.text(sql)):
pokestops.append({
"id": row[0],
"name": row[1],
"image": row[2],
"latitude": row[3],
"longitude": row[4],
"date_lure_expiration": row[5],
"date_change": row[6]
})
return pokestops
def _database_fetch_gyms(self, session=None, ne_latitude=0, ne_longitude=0, sw_latitude=0, sw_longitude=0):
gyms = []
sql = """
SELECT
t.id as "team_id",
t.name as "team_name",
g.id as "gym_id",
g.name as "gym_name",
g.image_url as "gym_image",
g.prestige as "gym_prestige",
g.latitude as "latitude",
g.longitude as "longitude",
g.date_change as "date_change"
FROM
db_pokestats.team as t,
db_pokestats.gym as g
WHERE
t.id = g.cd_team
and g.latitude < {0}
and g.latitude > {1}
and g.longitude < {2}
and g.longitude > {3}
""".format(ne_latitude, sw_latitude, ne_longitude, sw_longitude)
for row in session.execute(sqlalchemy.text(sql)):
gyms.append({
"team_id": row[0],
"team_name": row[1],
"gym_id": row[2],
"gym_name": row[3],
"gym_image": row[4],
"gym_prestige": row[5],
"latitude": row[6],
"longitude": row[7],
"date_change": row[8]
})
return gyms
def _database_fetch_scanners(self, session=None, ne_latitude=0, ne_longitude=0, sw_latitude=0, sw_longitude=0):
scanners = []
sql = """
SELECT
s.id as "id",
sl.latitude as "latitude",
sl.longitude as "longitude",
round(sl.distance*1000) as "distance",
s.date_change as "date_change"
FROM
db_pokestats.scanner as s,
db_pokestats.scanner_location as sl,
db_pokestats.scanner_account_statistic as sas
WHERE
s.cd_location = sl.id
and sas.cd_account = s.cd_account
and sl.latitude < {0}
and sl.latitude > {1}
and sl.longitude < {2}
and sl.longitude > {3}
""".format(ne_latitude, sw_latitude, ne_longitude, sw_longitude)
for row in session.execute(sqlalchemy.text(sql)):
scanners.append({
"id": row[0],
"latitude": row[1],
"longitude": row[2],
"distance": row[3],
"date_change": row[4]
})
return scanners
def index(self):
if request.args.get('latitude') and request.args.get('longitude'):
pos_latitude = request.args.get('latitude')
pos_longitude = request.args.get('longitude')
pos_gps=0
else:
pos_latitude = self.conf_latitude
pos_longitude = self.conf_longitude
pos_gps=1
return render_template('map.html',
lat=pos_latitude,
lng=pos_longitude,
gps=pos_gps,
gmaps_key=self.conf_gmapkey)
def json_put_user_geo(self):
json_dict = {
"saved": "true",
}
return jsonify(json_dict)
def json_get_data(self):
json_dict = {
"scanned": [],
"gyms": [],
"pokestops": [],
"pokemons": []
}
session = self._database_init()
if request.args.get('pokemon') == "true":
json_dict['pokemons'] = self._database_fetch_pokemons(session=session, ne_latitude=request.args.get('ne_latitude'), ne_longitude=request.args.get('ne_longitude'), sw_latitude=request.args.get('sw_latitude'), sw_longitude=request.args.get('sw_longitude'))
if request.args.get('pokestops') == "true":
json_dict['pokestops'] = self._database_fetch_pokestops(session=session, ne_latitude=request.args.get('ne_latitude'), ne_longitude=request.args.get('ne_longitude'), sw_latitude=request.args.get('sw_latitude'), sw_longitude=request.args.get('sw_longitude'))
if request.args.get('gyms') == "true":
json_dict['gyms'] = self._database_fetch_gyms(session=session, ne_latitude=request.args.get('ne_latitude'), ne_longitude=request.args.get('ne_longitude'), sw_latitude=request.args.get('sw_latitude'), sw_longitude=request.args.get('sw_longitude'))
if request.args.get('scanned') == "true":
json_dict['scanned'] = self._database_fetch_scanners(session=session, ne_latitude=request.args.get('ne_latitude'), ne_longitude=request.args.get('ne_longitude'), sw_latitude=request.args.get('sw_latitude'), sw_longitude=request.args.get('sw_longitude'))
self._database_close(session)
return jsonify(json_dict)
def json_put_data_geo(self):
json_dict = {
"saved": False,
"latitude": self.conf_latitude,
"longitude": self.conf_longitude
}
if request.args.get('latitude') and request.args.get('longitude'):
json_dict['saved'] = True
json_dict['latitude'] = request.args.get('latitude')
json_dict['longitude'] = request.args.get('longitude')
return jsonify(json_dict)
class SchemaDisplay:
"""
Класс визуализации модели базы данных на основе ORM SQLAlchemy.
Для работы необходимы: ``pydot``, ``graphviz``.
Параметры конфигурационного файла:
.. code-block:: xml
:linenos:
<?xml version="1.0" encoding="UTF-8"?>
<configuration>
<sphinx>
<AutoDocSchemaDisplay path_graph="$path/share/sphinx/static/schema_graph.png" />
</sphinx>
</configuration>
"""
def __init__(self):
self.config = Config()
self.path_graph = self.config.get("sphinx", "AutoDocSchemaDisplay", "path_graph", "")
# Инициализация параметров логирования.
logging.basicConfig(level=int(self.config.get("logging", "console", "level", "10")), stream=sys.stdout,
format='%(asctime)s [%(module)15s] [%(funcName)19s] [%(lineno)4d] [%(levelname)7s] [%(threadName)4s] %(message)s')
def render_schema(self):
"""
Отрисовка структуры схемы БД.
Записывает построенный график в фаил.
:return: None
"""
from sqlalchemy_schemadisplay import create_uml_graph, create_schema_graph
from sqlalchemy.orm import class_mapper
log.info("Формирование списка таблиц.")
mappers = []
for schema_name in dir(Schema):
if schema_name[0] == '_':
continue
if schema_name in ['Base', 'log']:
continue
schema_class = getattr(Schema, schema_name)
if hasattr(schema_class, "__tablename__"):
log.debug(" Анализ схемы: {}".format(schema_name))
mappers.append(class_mapper(schema_class))
log.info("Формирование окружения GraphViz.")
graph = create_uml_graph(
mappers,
show_inherited=False,
show_operations=False,
show_datatypes=False,
show_multiplicity_one=False
)
log.info("Формирование графика.")
graph.write_png(self.path_graph)
log.info("Формирование графика завершено.")
def _thread_check(thread):
scanner_id = int(thread.name)
scanner = session_mysql.query(dbScanner).get(scanner_id)
# Проверка на только включенные при рабочем сервере
if thread.metrica.time_await + datetime.timedelta(
minutes=15) < datetime.datetime.now():
log.info("[{0} - Найден битый сканнер".format(scanner.id))
thread.join()
arguments = _arg_parse()
config = Config()
session_maker = init(config)
session_mysql = session_maker()
scanner_alive = True
if __name__ == '__main__':
path = os.path.abspath(
os.path.join(os.path.dirname(os.path.realpath(__file__)), '..'))
pid = str(os.getpid())
pidfile = "{0}/.log/worker.pid".format(path)
file(pidfile, 'w').write(pid)
logging.basicConfig(
filename="{0}/.log/worker.log".format(path),
filemode='w',