def deleteDocumet(client: firestore.Client, collection: str, document: str):
"""ドキュメントを削除する
Arguments:
client {firestore.Client} -- [description]
collection {str} -- [description]
document {str} -- [description]
"""
client.collection(collection).document(document).delete()
def addDocument(client: firestore.Client, collection: str, document: str,
content: str):
"""コレクションへドキュメントを登録する
Arguments:
client {firestore.Client} -- [description]
collection {str} -- collection name
document {str} -- document name
content {str} -- jsonライクな定義
"""
client.collection(collection).document(document).set(content)
def get_people_doc_refs(
db: firestore.Client, field_name: str, operator: str, field_value: Any
) -> Optional[List[DocumentSnapshot]]:
"""Query 'people' collection for a document whose 'field_name' has 'operator'
relation with 'field_value'.
Args:
field_name: the field to query for.
operator: determines the condition for matching ('==', ...).
field_value: the value that satisfies the condition.
Raises:
FirestoreConnectorError: if querying matching documents returned error code.
Caught by decorator to return error code.
Returns:
a list of matching document snapshots. None if no documents are found.
Examples:
>>> fc.get_people_doc_refs(db, "dealroom_id", "==", 1169416)[0].to_dict()
>>> fc.get_people_doc_refs(db, "linkedin", "array_contains", "https://www.linkedin.com/in/vess/")[0].to_dict()
"""
people_ref = db.collection("people")
matching_docs = [
doc for doc in _filtered_stream(people_ref, field_name, operator, field_value)
]
# This is super weird, why return None when you can return a PERFECTLY EMPTY list?
# Not changing it to avoid breaking-changes.
if len(matching_docs) == 0:
return None
return matching_docs
def getDocument(client: firestore.Client, collection: str):
"""Collectioからドキュメント一覧を取得する
Arguments:
client {firestore.Client} -- [description]
collection {str} -- [description]
Returns:
[type] -- [description]
"""
return client.collection(collection).stream()
class OldFirestoreWatcher:
game_commands_template = "commands/{game_id}/commands/"
def __init__(self, client: Optional[Client] = None):
self._watchers = {}
self._collections = defaultdict(dict)
if client is None:
self._client = Client()
else:
self._client = client
self._watchers_lock = Lock()
def __del__(self):
for colname, watcher in self._watchers:
watcher.unsubscribe()
def _watch_new_collection(self, collection):
with self._watchers_lock:
if collection not in self._watchers:
col_ref = self._client.collection(collection)
col_cb = partial(self._collection_callback,
collection=collection)
self._watchers[collection] = col_ref.on_snapshot(col_cb)
# noinspection PyUnusedLocal
def _collection_callback(
self,
docs: List[DocumentSnapshot],
changes: List[DocumentChange],
read_time: datetime,
collection: str,
):
col_dict = self._collections[collection]
for change in changes:
if change.type in [ChangeType.ADDED, ChangeType.MODIFIED]:
try:
chat_command = ChatCommand(**change.document.to_dict())
except ValidationError:
print(f"Error parsing change for doc {change.document.id}")
continue
col_dict[change.document.id] = chat_command
elif change.type == ChangeType.REMOVED:
col_dict.pop(change.document.id)
def get_cmd(self, cmd_name: str, game_id: int = "global"):
pass
def get_cmds_for_game(self, collection: Optional[int]):
pass
def authenticate(
username: str,
password: str,
collection: str = "users",
db: firestore.Client = None,
) -> bool:
"""A function to authenticate with a Firestore database.
Args:
username (str): The requester's username.
password (str): The requester's password.
collection (str): Keyword argument. Represents the
the name of the collection to query.
db (google.cloud.firestore.Client): Keyword argument. A custom client
that can be used for testing.
Returns:
bool: True if successful, False otherwise.
:ret type: bool
"""
if db is None:
db = firestore.Client()
user_ref = (db.collection(collection).where("username", "==",
username).stream())
user_result = [user.to_dict() for user in user_ref]
if len(user_result) > 1:
logging.debug("Username and password combo not found in database.")
return False
if user_result[0]["password"] == password:
return True
else:
logging.debug("Username and password combo not found in database.")
return False
def set_status_to_down(db: firestore.Client, db_type: str, db_size: str,
resource_id: str):
pool_ref = (db.collection("db_resources").document(db_type).collection(
"sizes").document(db_size).collection("resources"))
pool_ref.document(resource_id).update({"status": "down"})
class DynamicoListener():
"""
- iReCHeck class to listen to the dynamico firebase and publish the changes in the ROS node
- Give the name of the collection you want to listen in the "listening_collection" in the class constructor
- NOTE-> Google's server time is different from Switzerland timezone
"""
def __init__(self, listening_collection_1):#, listening_collection_2):
# initialize ROS node
rospy.init_node('dynamicolistener', anonymous=True)
# initialize publishers/subscribers
# rospy.Subscriber([topic_name],[topic_type],[callback_function_name])
# rospy.Publisher([topic_name],[topic_type],[max_queue_size])
self.pubMsg = rospy.Publisher('dynamicomsg', String, queue_size=10)
# get credentials information from the file
path = os.path.realpath(__file__).replace('dynamicoListener.py','')
with open(path + '/DYNAMICO_CREDENTIALS.txt') as f:
# with open(path + '/DYNAMICO_CREDENTIALS_PARIS.txt') as f:
data = json.load(f)
print("DATA", data)
self.email = data['EMAIL']
self.password = data['PASSWORD']
self.API_KEY = data['API_KEY']
self.user_id = data['USER_ID']
self.project_id = data['PROJECT_ID']
# sign in on the Dynamico Firestore as done in https://gist.github.com/Bob-Thomas/4d9370c6b5432fb5150d3618e0ae71ba
self.FIREBASE_REST_API = "https://identitytoolkit.googleapis.com/v1/accounts"
response = self.sign_in_with_email_and_password(self.FIREBASE_REST_API, self.API_KEY, self.email, self.password)
self.listening_collection = listening_collection_1
# use google.oauth2.credentials and the response object to create the correct user credentials
creds = Credentials(response['idToken'], response['refreshToken'])
self.db = Client(self.project_id, creds)
# create an event to be notified of changes in the Dynamico Firestore
self.dynamicoCallback = threading.Event()
# watch the changes in the listening_collection only with regards to our user
doc_ref = self.db.collection(listening_collection_1).where(u'userId', u'==', self.user_id)
self.doc_watch_1 = doc_ref.on_snapshot(self.on_snapshot_1)
# watch the changes in the listening_collection only with regards to our user
# doc_ref_2 = self.db.collection(listening_collection_2).where(u'userId', u'==', self.user_id)
# self.doc_watch_2 = doc_ref.on_snapshot(self.on_snapshot_1)
# [DEBUG ONLY]
print("Waiting for messages")
# # keep python from exiting until this node is stopped
# rospy.spin()
# we use the sign_in_with_email_and_password function from https://gist.github.com/Bob-Thomas/49fcd13bbd890ba9031cc76be46ce446
def sign_in_with_email_and_password(self, url, api_key, email, password):
request_url = "%s:signInWithPassword?key=%s" % (url, api_key)
headers = {"content-type": "application/json; charset=UTF-8"}
data = json.dumps({"email": email, "password": password, "returnSecureToken": True})
resp = requests.post(request_url, headers=headers, data=data)
# check for errors
try:
resp.raise_for_status()
# [DEBUG ONLY]
# print(resp)
except HTTPError as e:
raise HTTPError(e, resp.text)
return resp.json()
def on_snapshot_1(self, doc_snapshot, changes, read_time):
print("LEN " + self.listening_collection + " : ", len(changes))
if len(changes)==1:
# if True:
# [DEBUG ONLY]
# print("\n\nChanges after initialized --------------\n")
item = changes[0].document._data #ch.document._data
# correct Google's stupidity of DateTime with Nanoseconds .... (why god???)
# NOTE again -> Google's server time is different from Switzerland timezone
val1 = item['createdAt']
year,month,day,hour,minute,second,tzinfo = val1.year,val1.month,val1.day,val1.hour, val1.minute, val1.second, val1.tzinfo
utc_time = "%s-%s-%s %s:%s:%s"%(year, month, day,hour,minute,second)
item['createdAt']=utc_time
final = pd.DataFrame(item, index=[0])
# convert the dataframe in a json string and publish it as a ROS message
msg = final.to_json(orient='records')
rospy.loginfo(msg)
self.pubMsg.publish(msg)
#self.dynamicoCallback.set()
# score = item['score']
# mymsg = "Foing to next phase! "#You scored {} points in the game {}".format(item['score'], item['game'])
# msg_2_pub = f''' rostopic pub -1 /qt_robot/speech/say std_msgs/String "data: '{mymsg}'" '''
# os.system(msg_2_pub)
# msg_2_pub = f''' rosservice call /qt_robot/behavior/talkText "message: '{mymsg}'" '''
# rosservice call /qt_robot/behavior/talkText "message: 'I am Q.T.'"
# print ("MSG HERE", msg_2_pub)
# [DEBUG ONLY]
# print("\nEND of collection----------------\n\n")
else:
# [DEBUG ONLY]
print("\n\n-------------- Fisrt time. No message was sent --------------\n")
# create a callback on_snapshot function to capture changes in the Dynamico Firestore
def on_snapshot_OLD(self, doc_snapshot, changes, read_time):
# [DEBUG ONLY]
# print("\n\nPublication from collection-> '{}' --------------\n".format(self.listening_collection))
# create an empty list of dataframes to be populated with the changes in database and sent over ROS
final = []
if len(changes)>1:
# [DEBUG ONLY]
print("\n\nFisrt time '{}' --------------\n".format(len(changes)))
key_list = list(changes[0].document._data.keys())
final = pd.DataFrame(columns=key_list)
# item = ch.document._data
# final.append(pd.DataFrame(item, index=[0]))
# final =
else:
for ch in changes:
# get the item as a dictionary
item = ch.document._data
# correct Google's stupidity of DateTime with Nanoseconds .... (why god???)
# NOTE again -> Google's server time is different from Switzerland timezone
val1 = item['createdAt']
year,month,day,hour,minute,second,tzinfo = val1.year,val1.month,val1.day,val1.hour, val1.minute, val1.second, val1.tzinfo
utc_time = "%s-%s-%s %s:%s:%s"%(year, month, day,hour,minute,second)
item['createdAt']=utc_time
#create a pandas dataframe with the current information or change in the firebase
final.append(pd.DataFrame(item, index=[0]))
# concatenate all the new changes in a single dataframe
final = pd.concat(final, ignore_index=True)
final = final.sort_values('createdAt', ignore_index=True)
# convert the dataframe in a json string and publish it as a ROS message
msg = final.to_json(orient='records')
rospy.loginfo(msg)
self.pubMsg.publish(msg)
#self.dynamicoCallback.set()
# [DEBUG ONLY]
print("\nEND of collection----------------\n\n")
# test quering the dynamico db
def test(self):
# diagnosesDocs = self.db.collection(u'diagnoses').where(u'userId', u'==', u"TXGCcUC6u5duIO6VRhfnYAXXwTg2").stream()
# diagnosesDocs = self.db.collection(u'diagnoses').where(u'userId', u'==', u"TXGCcUC6u5duIO6VRhfnYAXXwTg2").stream()
diagnosesDocs = self.db.collection('scores').where(u'userId', u'==', self.user_id) #.where(u'childId', u'==', u"75DA34CC-64C4-40C5-8639-6653FBF26FDA")
# print(type(diagnosesDocs))
# convert the retrieved docs in dictionaries and extract the handwriting scores values
for doc in diagnosesDocs.stream():
tempDict = doc.to_dict()
# scores = [-1, -1, -1, -1, -1]
# scores[0] = tempDict[u'kinematicScore']
# scores[1] = tempDict[u'pressureScore']
# scores[2] = tempDict[u'staticScore']
# scores[3] = tempDict[u'tiltScore']
# scores[4] = tempDict[u'totalScore']
print(tempDict)
print("\n")
def set_people_doc_ref(
db: firestore.Client,
field_name: str,
operator: str,
field_value,
payload: dict,
) -> StatusCode:
"""Updates or Creates a single document from 'people' collection with 'payload' where 'field_name' has 'operator'
relation with 'field_value'.
Args:
field_name: the field to query for.
operator: determines the condition for matching ('==', ...).
field_value: the value that satisfies the condition.
payload: The actual data that the newly created document will have OR the fields to update.
Returns:
integer signifying what the status of the operation is.
Examples:
>>> fc.set_people_doc_ref(db, "dealroom_id", "==", 1003809000, {"foo":"bar"})
>>> fc.set_people_doc_ref(db, "linkedin", "array_contains", "https://www.linkedin.com/in/vess/", {"foo":["bar",2]})
"""
people_collection_ref = db.collection("people")
_payload = {**payload}
people_refs = (
get_people_doc_refs(db, field_name, operator, field_value)
if field_name and operator and field_value
else []
)
matching_docs = len(people_refs) if people_refs else 0
# CREATE: If there are not matching documents in people
if matching_docs == 0:
# Add any default values to the payload if non are already there
if "dealroom_id" not in _payload and "dealroom_uuid" not in _payload:
_payload["dealroom_id"] = DealroomEntity.NOT_IN_DB.value
_payload["dealroom_uuid"] = DealroomEntity.NOT_IN_DB.value
# Validate that the new document will have the minimum required fields
try:
_validate_new_people_doc_payload(_payload)
except (ValueError, KeyError) as ex:
# TODO: raise Custom Exception (DN-932: https://dealroom.atlassian.net/browse/DN-932)
logging.error(ex)
return StatusCode.ERROR
people_doc_ref = people_collection_ref.document()
operation_status_code = StatusCode.CREATED
# UPDATE:
elif matching_docs == 1:
# Ensure that dealroom_id is type of number if it appears on the payload
if "dealroom_id" in _payload:
_validate_dealroom_id(_payload["dealroom_id"])
people_doc_ref = people_refs[0].reference
operation_status_code = StatusCode.UPDATED
# If more than one document were found then it's an error.
else:
# TODO: Raise a Custom Exception (DuplicateDocumentsException) with the same message when we replace ERROR constant with actual exceptions
# (DN-932: https://dealroom.atlassian.net/browse/DN-932)
logging.error("Found more than one documents to update for this payload")
return StatusCode.ERROR
res = set(people_doc_ref, _payload)
if res == StatusCode.ERROR:
# TODO: Raise a Custom Exception (FirestoreException) with the same message when we replace ERROR constant with actual exceptions
# (DN-932: https://dealroom.atlassian.net/browse/DN-932)
logging.error(
f"Couldn't `set` document {people_doc_ref.id}. Please check logs above."
)
return StatusCode.ERROR
return operation_status_code
def set_history_doc_refs(
db: firestore.Client, payload: dict, finalurl_or_dealroomid: str = None
) -> StatusCode:
"""Updates or creates a document in history collection
Args:
db: the client that will perform the operations.
payload: The actual data that the newly created document will have or
the fields to update. Any of 'final_url', 'dealroom_id' or
'dealroom_uuid' is required to find the correct document to set.
finalurl_or_dealroomid: either a domain, a dealroom ID or a dealroom UUID.
Query documents that match this parameter.
Returns:
integer code to signify what operation was carried out.
Examples:
>>> db = new_connection(project=FIRESTORE_PROJECT_ID)
>>> set_history_refs(db, {"final_url": "dealroom.co", "dealroom_id": "1111111")
"""
history_col = db.collection(HISTORY_COLLECTION_PATH)
_payload = {**payload}
history_refs = {}
# lookup for the document using both identifiers, final_url & dealroom_id
final_url, dealroom_id = _get_final_url_and_dealroom_id(
payload, finalurl_or_dealroomid
)
if isinstance(dealroom_id, DealroomIdentifier):
value = dealroom_id.value
else:
value = dealroom_id
history_refs = get_history_doc_refs(db, final_url, value)
if history_refs == StatusCode.ERROR:
# TODO: raise Custom Exception (DN-932: https://dealroom.atlassian.net/browse/DN-932)
return StatusCode.ERROR
operation_status_code = StatusCode.ERROR
key_found = None
if "dealroom_id" in history_refs and len(history_refs["dealroom_id"]) > 0:
count_history_refs = len(history_refs["dealroom_id"])
key_found = "dealroom_id"
elif "dealroom_id_old" in history_refs and len(history_refs["dealroom_id_old"]) > 0:
count_history_refs = len(history_refs["dealroom_id_old"])
key_found = "dealroom_id_old"
elif "dealroom_uuid" in history_refs and len(history_refs["dealroom_uuid"]) > 0:
count_history_refs = len(history_refs["dealroom_uuid"])
key_found = "dealroom_uuid"
elif (
"dealroom_uuid_old" in history_refs
and len(history_refs["dealroom_uuid_old"]) > 0
):
count_history_refs = len(history_refs["dealroom_uuid_old"])
key_found = "dealroom_uuid_old"
elif "final_url" in history_refs and len(history_refs["final_url"]) > 0:
count_history_refs = len(history_refs["final_url"])
key_found = "final_url"
elif (
"current_related_urls" in history_refs
and len(history_refs["current_related_urls"]) > 0
):
count_history_refs = len(history_refs["current_related_urls"])
key_found = "current_related_urls"
else:
count_history_refs = 0
document_matches_by_final_url = key_found == "final_url"
if document_matches_by_final_url and isinstance(dealroom_id, DealroomIdentifier):
count_history_refs = check_for_deleted_profiles(
history_refs[key_found], dealroom_id, count_history_refs
)
count_history_refs = check_for_in_progress_profiles(
history_refs[key_found], dealroom_id, count_history_refs
)
# CREATE: If there are not available documents in history
if count_history_refs <= 0:
if isinstance(dealroom_id, DealroomIdentifier):
# TODO: Add UUID/ID depending on what identifier was passed (DS2-285: https://dealroom.atlassian.net/browse/DS2-285)
_payload = {
dealroom_id.field_name: dealroom_id.value,
"final_url": "",
**payload,
}
else:
_payload = {
"dealroom_id": DealroomEntity.NOT_IN_DB.value,
"dealroom_uuid": DealroomEntity.NOT_IN_DB.value,
"final_url": finalurl_or_dealroomid,
**payload,
}
# Validate that the new document will have the minimum required fields
try:
_validate_new_history_doc_payload(_payload)
except (ValueError, KeyError) as ex:
# TODO: raise Custom Exception (DN-932: https://dealroom.atlassian.net/browse/DN-932)
logging.error(ex)
return StatusCode.ERROR
history_ref = history_col.document()
operation_status_code = StatusCode.CREATED
# UPDATE:
elif count_history_refs == 1:
try:
_validate_update_history_doc_payload(_payload)
except ValueError as ex:
# TODO: raise Custom Exception (DN-932: https://dealroom.atlassian.net/browse/DN-932)
logging.error(ex)
return StatusCode.ERROR
history_ref = history_refs[key_found][0]
operation_status_code = StatusCode.UPDATED
# If more than one document were found then it's an error.
else:
# TODO: Raise a Custom Exception (DuplicateDocumentsException) with the same message when we replace ERROR constant with actual exceptions
# (DN-932: https://dealroom.atlassian.net/browse/DN-932)
logging.error("Found more than one documents to update for this payload")
return StatusCode.ERROR
# Ensure that dealroom_id is type of number
if "dealroom_id" in _payload:
_payload["dealroom_id"] = int(_payload["dealroom_id"])
res = set(history_ref, _payload)
if res == StatusCode.ERROR:
# TODO: Raise a Custom Exception (FirestoreException) with the same message when we replace ERROR constant with actual exceptions
# (DN-932: https://dealroom.atlassian.net/browse/DN-932)
logging.error(
f"Couldn't `set` document {finalurl_or_dealroomid}. Please check logs above."
)
return StatusCode.ERROR
return operation_status_code
def get_history_doc_refs(
db: firestore.Client,
final_url: Optional[str] = None,
dealroom_id: Union[int, str, None] = None,
) -> Dict[str, List[DocumentReference]]:
"""Match documents on certain fields and return, for each field matched, the
matching document refs.
Args:
db: the client that will perform the operations.
final_url: A domain. Query documents that match this parameter on fields
"final_url" and "current_related_urls".
dealroom_id: A dealroom ID or UUID. Query documents that match this
parameter on fields "dealroom_id" and "dealroom_id_old", or on
"dealroom_uuid" and "dealroom_uuid_old" respectively.
Raises:
FirestoreConnectorError: if querying matching documents returned error code.
Caught by decorator to return error code.
Returns:
a dictionary made of lists of document references matching the input
parameter (the values). The keys indicate which fields were matched:
any of final_url, current_related_urls, dealroom_id, dealroom_id_old
or dealroom_uuid, dealroom_uuid_old.
Examples:
>>> db = new_connection(project=FIRESTORE_PROJECT_ID)
>>> doc_refs = get_history_refs(db, "dealroom.co")
"""
if not final_url and not dealroom_id:
# TODO: raise Custom Exception (DN-932: https://dealroom.atlassian.net/browse/DN-932)
logging.error(
"Any of `final_url` or `dealroom_id` need to be used as a unique identifier"
)
return StatusCode.ERROR
try:
dr_id = determine_identifier(dealroom_id)
except InvalidIdentifier as exc:
dr_id = None
history_ref = db.collection(HISTORY_COLLECTION_PATH)
result = {}
# Add results for matched documents over `dealroom_id`
if dr_id:
result[dr_id.field_name] = _filtered_stream_refs(
history_ref, dr_id.field_name, "==", dr_id.value
)
result[dr_id.field_name_old] = _filtered_stream_refs(
history_ref, dr_id.field_name_old, "==", dr_id.value
)
# Add results for matched documents over `final_url`
if final_url:
# Extract the final_url in the required format, so we can query the collection with the exact match.
try:
website_url = extract(final_url)
except InvalidURLFormat as exc:
# TODO: raise Custom Exception (DN-932: https://dealroom.atlassian.net/browse/DN-932)
logging.error(f"'final_url': {final_url} is not a valid url: {exc}")
return StatusCode.ERROR
result["final_url"] = _filtered_stream_refs(
history_ref, "final_url", "==", website_url
)
result["current_related_urls"] = _filtered_stream_refs(
history_ref, "current_related_urls", "array_contains", website_url
)
return result