def upload_new_resources(new_resources_df: pd.DataFrame,
firestore_resources: Dict[Resource, str], db,
sheet: gspread.models.Worksheet) -> List[int]:
length = len(new_resources_df.index)
log(f"{length} resources to upload")
added = 0
uploaded_rows = list()
for index, row in new_resources_df.iterrows():
links = Links(row["card link"], row["website"])
resource = Resource(row["resource name"], True, row["description"],
row["image link"], row["category"],
row["tags"].split(", "), links)
try:
if resource not in firestore_resources:
db.collection(FIREBASE_COLLECTION).add(resource.to_dict())
log(f"\tAdded {row['resource name']} to {FIREBASE_COLLECTION}")
else:
db.collection(FIREBASE_COLLECTION).document(
firestore_resources[resource]).set(resource.to_dict())
log(f"\tUpdated {row['resource name']} in {FIREBASE_COLLECTION}"
)
except:
log(f"Error uploading data to firestore. {added} / {length} resources uploaded successfully"
)
return uploaded_rows
added += 1
uploaded_rows.append(index + 1)
log(f"\nAdded {added} / {length} entries to Firestore")
return uploaded_rows
def init_online_data(self):
print('Start to trace the ' +
UseStyle('ONLINE', fore='green', mode='underline') + ' data')
self.device_config_obj = DeviceConfigs.initialize(self.mars_config)
self.hosts_obj = Hosts.initialize(self.mars_config)
self.group_obj = Groups.initialize(self.mars_config)
self.flow_obj = Flows.initialize(self.mars_config)
self.link_obj = Links.initialize(self.mars_config)
def upload_new_resources(new_resources_df: pd.DataFrame,
firestore_resources: Dict[Resource, str], db,
sheet: gspread.models.Worksheet) -> List[int]:
length = len(new_resources_df.index)
log(f"{length} resources to upload")
added = 0
uploaded_rows = list()
for index, row in new_resources_df.iterrows():
links = Links(row["card link"], row["website"])
date_created = datetime.now().strftime("%Y/%m/%d %H:%M:%S")
resource = Resource(title=row["resource name"],
reviewed=True,
want_support_with=row["want support with"],
this_resource_offers=row["this resource offers"],
description=row["description"],
img=row["image link"],
category=row["category"],
tags=row["tags"].split(", "),
links=links,
date_created=date_created,
ranking=row["ranking"])
try:
category_document = db.collection(FIREBASE_COLLECTION).document(
resource.category.replace("/ ", "_"))
if resource not in firestore_resources:
category_document.update({
"resource_list":
firestore.ArrayUnion([resource.title])
}) # Update resource list
category_document.update(
{"tag_list":
firestore.ArrayUnion(resource.tags)}) # Update tag list
category_document.collection("resources").add(
resource.to_dict()) # Add new document to collection
log(f"\tAdded {row['resource name']} to {FIREBASE_COLLECTION}/{category_document.id}"
)
else:
resource._date_created = category_document.collection(
"resources").document(firestore_resources[resource]).get(
).to_dict()["dateCreated"]
category_document.collection("resources").document(
firestore_resources[resource]).set(resource.to_dict(
)) # Update old document in collection
log(f"\tUpdated {row['resource name']} in {FIREBASE_COLLECTION}/{category_document.id}"
)
except Exception as e:
log(f"Error uploading data to firestore. {added} / {length} resources uploaded successfully"
)
print(e)
return uploaded_rows
added += 1
uploaded_rows.append(index + 1)
log(f"\nAdded {added} / {length} entries to Firestore")
return uploaded_rows
def show_online_links(self):
print_normal('Show the ' +
UseStyle('ONLINE', fore='green', mode='underline') +
' links.')
link_obj = Links.initialize(mars_config=self.mars_config)
devices_config_obj = DeviceConfigs.initialize(
mars_config=self.mars_config)
for device_config in devices_config_obj.get_data():
print_normal_start('Device Name : ' + device_config['name'],
color='yellow')
for link in link_obj.get_data():
if link['src']['device'] == device_config['id']:
print_normal_center(
link_to_line_string(link, devices_config_obj))
print_normal_end('')
print_normal('')
def init_snap_data(self, snap_time_str):
print('Start to trace the ' +
UseStyle(snap_time_str, fore='green', mode='underline') +
' snap data\n')
snap_time = format_time_string_2_number(snap_time_str)
self.snap_time = snap_time
self.device_config_obj = DeviceConfigs.initialize_with(
self.mars_config, get_devices_configs(self.mars_config, snap_time))
self.hosts_obj = Hosts.initialize_with(
self.mars_config, get_host(self.mars_config, snap_time))
self.group_obj = Groups.initialize_with(
self.mars_config, get_group(self.mars_config, snap_time))
self.flow_obj = Flows.initialize_with(
self.mars_config, get_flow(self.mars_config, snap_time))
self.link_obj = Links.initialize_with(
self.mars_config, get_link(self.mars_config, snap_time))
def show_snap_links(self, snap_time_str):
snap_time = format_time_string_2_number(snap_time_str)
print_normal('Show the ' +
UseStyle(snap_time_str, fore='green', mode='underline') +
' links.')
link_obj = Links.initialize_with(self.mars_config,
get_link(self.mars_config, snap_time))
devices_config_obj = DeviceConfigs.initialize_with(
self.mars_config, get_devices_configs(self.mars_config, snap_time))
for device_config in devices_config_obj.get_data():
print_normal_start('Device Name : ' + device_config['name'],
color='yellow')
for link in link_obj.get_data():
if link['src']['device'] == device_config['id']:
print_normal_center(
link_to_line_string(link, devices_config_obj))
print_normal_end('')
print_normal('')
def snap_all_data(self):
print_normal('Going to snap all the data from ' +
self.mars_config.get_url())
device_object = Devices.initialize(self.mars_config)
device_config_object = DeviceConfigs.initialize(self.mars_config)
host_object = Hosts.initialize(self.mars_config)
flow_object = Flows.initialize(self.mars_config)
group_object = Groups.initialize(self.mars_config)
link_object = Links.initialize(self.mars_config)
t = int(time.time())
data_path = self.mars_config.get_base_path() + '/' + str(t)
os.mkdir(data_path)
device_object.save(data_path)
device_config_object.save(data_path)
host_object.save(data_path)
flow_object.save(data_path)
group_object.save(data_path)
link_object.save(data_path)
print_normal('Success to snap the data')
resources_df = resources_df[resources_df["category"].notna()]
resources_df = resources_df.replace(np.nan, "", regex=True)
resources_df = resources_df.applymap(str)
resources_df["category"] = resources_df["category"].str.lower()
resources_df["tags"] = resources_df["tags"].str.lower()
resources_df["reviewed"] = resources_df["reviewed"].str.lower()
resources_df["reviewed"] = resources_df["reviewed"].map({
"true": True,
"false": False
})
# Initailize Cloud Firestore
log("\nInitializing Cloud Firestore with key " + service_key)
cred = credentials.Certificate(service_key)
firebase_admin.initialize_app(cred)
db = firestore.client()
# Add data to Firestore
log("\nAdding data to firestore...")
added = 0
for index, row in resources_df.iterrows():
links = Links(row["card link"], row["website"])
resource = Resource(row["resource name"], True, row["description"],
row["image link"], row["category"],
row["tags"].split(", "), links)
path = "resources"
db.collection(path).add(resource.to_dict())
added += 1
log("\tAdded " + row["resource name"] + " to " + path)
log("\nAdded " + str(added) + " entries to Firestore")
resources_df = resources_df.replace(np.nan, "", regex=True)
resources_df = resources_df.applymap(str)
resources_df["category"] = resources_df["category"].str.lower()
resources_df["tags"] = resources_df["tags"].str.lower()
resources_df["reviewed"] = resources_df["reviewed"].str.lower()
resources_df["reviewed"] = resources_df["reviewed"].map({
"true": True,
"false": False
})
# Initailize Cloud Firestore
log("\nInitializing Cloud Firestore with key " + service_key)
cred = credentials.Certificate(service_key)
firebase_admin.initialize_app(cred)
db = firestore.client()
# Add data to Firestore
log("\nAdding data to firestore...")
added = 0
for index, row in resources_df.iterrows():
links = Links(row["android link"], row["card link"], row["facebook"],
row["ios link"], row["website"])
resource = Resource(row["resource name"], True, row["description"],
row["image link"], row["category"],
row["tags"].split(", "), links)
path = "resources"
db.collection(path).add(resource.to_dict())
added += 1
log("\tAdded " + row["resource name"] + " to " + path)
log("\nAdded " + str(added) + " entries to Firestore")