def __init__(self, *args, **kwargs):
self.dbcon = DbConnector(mongo_url=self.url,
database_name=self.database,
table_name=self.table_name)
super(ProcessEventHub, self).__init__(*args, **kwargs)
class AvalonRestApi(RestApi):
dbcon = DbConnector(os.environ["AVALON_MONGO"], os.environ["AVALON_DB"])
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.dbcon.install()
@RestApi.route("/projects/<project_name>",
url_prefix="/avalon",
methods="GET")
def get_project(self, request):
project_name = request.url_data["project_name"]
if not project_name:
output = {}
for project_name in self.dbcon.tables():
project = self.dbcon[project_name].find_one(
{"type": "project"})
output[project_name] = project
return CallbackResult(data=self.result_to_json(output))
project = self.dbcon[project_name].find_one({"type": "project"})
if project:
return CallbackResult(data=self.result_to_json(project))
abort(404,
"Project \"{}\" was not found in database".format(project_name))
@RestApi.route("/projects/<project_name>/assets/<asset>",
url_prefix="/avalon",
methods="GET")
def get_assets(self, request):
_project_name = request.url_data["project_name"]
_asset = request.url_data["asset"]
if not self.dbcon.exist_table(_project_name):
abort(
404, "Project \"{}\" was not found in database".format(
project_name))
if not _asset:
assets = self.dbcon[_project_name].find({"type": "asset"})
output = self.result_to_json(assets)
return CallbackResult(data=output)
# identificator can be specified with url query (default is `name`)
identificator = request.query.get("identificator", "name")
asset = self.dbcon[_project_name].find_one({
"type": "asset",
identificator: _asset
})
if asset:
id = asset["_id"]
asset["_id"] = str(id)
return asset
abort(
404, "Asset \"{}\" with {} was not found in project {}".format(
_asset, identificator, project_name))
def result_to_json(self, result):
""" Converts result of MongoDB query to dict without $oid (ObjectId)
keys with help of regex matching.
..note:
This will convert object type entries similar to ObjectId.
"""
bson_json = bson.json_util.dumps(result)
# Replace "{$oid: "{entity id}"}" with "{entity id}"
regex1 = '(?P<id>{\"\$oid\": \"[^\"]+\"})'
regex2 = '{\"\$oid\": (?P<id>\"[^\"]+\")}'
for value in re.findall(regex1, bson_json):
for substr in re.findall(regex2, value):
bson_json = bson_json.replace(value, substr)
return json.loads(bson_json)
class ProcessEventHub(SocketBaseEventHub):
hearbeat_msg = b"processor"
url, database, table_name = get_ftrack_event_mongo_info()
is_table_created = False
pypelog = Logger().get_logger("Session Processor")
def __init__(self, *args, **kwargs):
self.dbcon = DbConnector(mongo_url=self.url,
database_name=self.database,
table_name=self.table_name)
super(ProcessEventHub, self).__init__(*args, **kwargs)
def prepare_dbcon(self):
try:
self.dbcon.install()
self.dbcon._database.list_collection_names()
except pymongo.errors.AutoReconnect:
self.pypelog.error(
"Mongo server \"{}\" is not responding, exiting.".format(
os.environ["AVALON_MONGO"]))
sys.exit(0)
except pymongo.errors.OperationFailure:
self.pypelog.error(
("Error with Mongo access, probably permissions."
"Check if exist database with name \"{}\""
" and collection \"{}\" inside.").format(
self.database, self.table_name))
self.sock.sendall(b"MongoError")
sys.exit(0)
def wait(self, duration=None):
"""Overriden wait
Event are loaded from Mongo DB when queue is empty. Handled event is
set as processed in Mongo DB.
"""
started = time.time()
self.prepare_dbcon()
while True:
try:
event = self._event_queue.get(timeout=0.1)
except queue.Empty:
if not self.load_events():
time.sleep(0.5)
else:
try:
self._handle(event)
self.dbcon.update_one(
{"id": event["id"]},
{"$set": {
"pype_data.is_processed": True
}})
except pymongo.errors.AutoReconnect:
self.pypelog.error(
("Mongo server \"{}\" is not responding, exiting."
).format(os.environ["AVALON_MONGO"]))
sys.exit(0)
# Additional special processing of events.
if event['topic'] == 'ftrack.meta.disconnected':
break
if duration is not None:
if (time.time() - started) > duration:
break
def load_events(self):
"""Load not processed events sorted by stored date"""
ago_date = datetime.datetime.now() - datetime.timedelta(days=3)
result = self.dbcon.delete_many({
"pype_data.stored": {
"$lte": ago_date
},
"pype_data.is_processed": True
})
not_processed_events = self.dbcon.find({
"pype_data.is_processed": False
}).sort([("pype_data.stored", pymongo.ASCENDING)])
found = False
for event_data in not_processed_events:
new_event_data = {
k: v
for k, v in event_data.items()
if k not in ["_id", "pype_data"]
}
try:
event = ftrack_api.event.base.Event(**new_event_data)
except Exception:
self.logger.exception(
L('Failed to convert payload into event: {0}', event_data))
continue
found = True
self._event_queue.put(event)
return found
def _handle_packet(self, code, packet_identifier, path, data):
"""Override `_handle_packet` which skip events and extend heartbeat"""
code_name = self._code_name_mapping[code]
if code_name == "event":
return
return super()._handle_packet(code, packet_identifier, path, data)
TOPIC_STATUS_SERVER,
TOPIC_STATUS_SERVER_RESULT)
from pype.ftrack.lib.custom_db_connector import DbConnector
from pypeapp import Logger
log = Logger().get_logger("Event storer")
subprocess_started = datetime.datetime.now()
class SessionFactory:
session = None
url, database, table_name = get_ftrack_event_mongo_info()
dbcon = DbConnector(mongo_url=url,
database_name=database,
table_name=table_name)
# ignore_topics = ["ftrack.meta.connected"]
ignore_topics = []
def install_db():
try:
dbcon.install()
dbcon._database.list_collection_names()
except pymongo.errors.AutoReconnect:
log.error("Mongo server \"{}\" is not responding, exiting.".format(
os.environ["AVALON_MONGO"]))
sys.exit(0)