def upload_new_resources(new_resources_df: pd.DataFrame, firestore_resources: Dict[Resource, str], db, sheet: gspread.models.Worksheet) -> List[int]: length = len(new_resources_df.index) log(f"{length} resources to upload") added = 0 uploaded_rows = list() for index, row in new_resources_df.iterrows(): links = Links(row["card link"], row["website"]) resource = Resource(row["resource name"], True, row["description"], row["image link"], row["category"], row["tags"].split(", "), links) try: if resource not in firestore_resources: db.collection(FIREBASE_COLLECTION).add(resource.to_dict()) log(f"\tAdded {row['resource name']} to {FIREBASE_COLLECTION}") else: db.collection(FIREBASE_COLLECTION).document( firestore_resources[resource]).set(resource.to_dict()) log(f"\tUpdated {row['resource name']} in {FIREBASE_COLLECTION}" ) except: log(f"Error uploading data to firestore. {added} / {length} resources uploaded successfully" ) return uploaded_rows added += 1 uploaded_rows.append(index + 1) log(f"\nAdded {added} / {length} entries to Firestore") return uploaded_rows
def init_online_data(self): print('Start to trace the ' + UseStyle('ONLINE', fore='green', mode='underline') + ' data') self.device_config_obj = DeviceConfigs.initialize(self.mars_config) self.hosts_obj = Hosts.initialize(self.mars_config) self.group_obj = Groups.initialize(self.mars_config) self.flow_obj = Flows.initialize(self.mars_config) self.link_obj = Links.initialize(self.mars_config)
def upload_new_resources(new_resources_df: pd.DataFrame, firestore_resources: Dict[Resource, str], db, sheet: gspread.models.Worksheet) -> List[int]: length = len(new_resources_df.index) log(f"{length} resources to upload") added = 0 uploaded_rows = list() for index, row in new_resources_df.iterrows(): links = Links(row["card link"], row["website"]) date_created = datetime.now().strftime("%Y/%m/%d %H:%M:%S") resource = Resource(title=row["resource name"], reviewed=True, want_support_with=row["want support with"], this_resource_offers=row["this resource offers"], description=row["description"], img=row["image link"], category=row["category"], tags=row["tags"].split(", "), links=links, date_created=date_created, ranking=row["ranking"]) try: category_document = db.collection(FIREBASE_COLLECTION).document( resource.category.replace("/ ", "_")) if resource not in firestore_resources: category_document.update({ "resource_list": firestore.ArrayUnion([resource.title]) }) # Update resource list category_document.update( {"tag_list": firestore.ArrayUnion(resource.tags)}) # Update tag list category_document.collection("resources").add( resource.to_dict()) # Add new document to collection log(f"\tAdded {row['resource name']} to {FIREBASE_COLLECTION}/{category_document.id}" ) else: resource._date_created = category_document.collection( "resources").document(firestore_resources[resource]).get( ).to_dict()["dateCreated"] category_document.collection("resources").document( firestore_resources[resource]).set(resource.to_dict( )) # Update old document in collection log(f"\tUpdated {row['resource name']} in {FIREBASE_COLLECTION}/{category_document.id}" ) except Exception as e: log(f"Error uploading data to firestore. {added} / {length} resources uploaded successfully" ) print(e) return uploaded_rows added += 1 uploaded_rows.append(index + 1) log(f"\nAdded {added} / {length} entries to Firestore") return uploaded_rows
def show_online_links(self): print_normal('Show the ' + UseStyle('ONLINE', fore='green', mode='underline') + ' links.') link_obj = Links.initialize(mars_config=self.mars_config) devices_config_obj = DeviceConfigs.initialize( mars_config=self.mars_config) for device_config in devices_config_obj.get_data(): print_normal_start('Device Name : ' + device_config['name'], color='yellow') for link in link_obj.get_data(): if link['src']['device'] == device_config['id']: print_normal_center( link_to_line_string(link, devices_config_obj)) print_normal_end('') print_normal('')
def init_snap_data(self, snap_time_str): print('Start to trace the ' + UseStyle(snap_time_str, fore='green', mode='underline') + ' snap data\n') snap_time = format_time_string_2_number(snap_time_str) self.snap_time = snap_time self.device_config_obj = DeviceConfigs.initialize_with( self.mars_config, get_devices_configs(self.mars_config, snap_time)) self.hosts_obj = Hosts.initialize_with( self.mars_config, get_host(self.mars_config, snap_time)) self.group_obj = Groups.initialize_with( self.mars_config, get_group(self.mars_config, snap_time)) self.flow_obj = Flows.initialize_with( self.mars_config, get_flow(self.mars_config, snap_time)) self.link_obj = Links.initialize_with( self.mars_config, get_link(self.mars_config, snap_time))
def show_snap_links(self, snap_time_str): snap_time = format_time_string_2_number(snap_time_str) print_normal('Show the ' + UseStyle(snap_time_str, fore='green', mode='underline') + ' links.') link_obj = Links.initialize_with(self.mars_config, get_link(self.mars_config, snap_time)) devices_config_obj = DeviceConfigs.initialize_with( self.mars_config, get_devices_configs(self.mars_config, snap_time)) for device_config in devices_config_obj.get_data(): print_normal_start('Device Name : ' + device_config['name'], color='yellow') for link in link_obj.get_data(): if link['src']['device'] == device_config['id']: print_normal_center( link_to_line_string(link, devices_config_obj)) print_normal_end('') print_normal('')
def snap_all_data(self): print_normal('Going to snap all the data from ' + self.mars_config.get_url()) device_object = Devices.initialize(self.mars_config) device_config_object = DeviceConfigs.initialize(self.mars_config) host_object = Hosts.initialize(self.mars_config) flow_object = Flows.initialize(self.mars_config) group_object = Groups.initialize(self.mars_config) link_object = Links.initialize(self.mars_config) t = int(time.time()) data_path = self.mars_config.get_base_path() + '/' + str(t) os.mkdir(data_path) device_object.save(data_path) device_config_object.save(data_path) host_object.save(data_path) flow_object.save(data_path) group_object.save(data_path) link_object.save(data_path) print_normal('Success to snap the data')
resources_df = resources_df[resources_df["category"].notna()] resources_df = resources_df.replace(np.nan, "", regex=True) resources_df = resources_df.applymap(str) resources_df["category"] = resources_df["category"].str.lower() resources_df["tags"] = resources_df["tags"].str.lower() resources_df["reviewed"] = resources_df["reviewed"].str.lower() resources_df["reviewed"] = resources_df["reviewed"].map({ "true": True, "false": False }) # Initailize Cloud Firestore log("\nInitializing Cloud Firestore with key " + service_key) cred = credentials.Certificate(service_key) firebase_admin.initialize_app(cred) db = firestore.client() # Add data to Firestore log("\nAdding data to firestore...") added = 0 for index, row in resources_df.iterrows(): links = Links(row["card link"], row["website"]) resource = Resource(row["resource name"], True, row["description"], row["image link"], row["category"], row["tags"].split(", "), links) path = "resources" db.collection(path).add(resource.to_dict()) added += 1 log("\tAdded " + row["resource name"] + " to " + path) log("\nAdded " + str(added) + " entries to Firestore")
resources_df = resources_df.replace(np.nan, "", regex=True) resources_df = resources_df.applymap(str) resources_df["category"] = resources_df["category"].str.lower() resources_df["tags"] = resources_df["tags"].str.lower() resources_df["reviewed"] = resources_df["reviewed"].str.lower() resources_df["reviewed"] = resources_df["reviewed"].map({ "true": True, "false": False }) # Initailize Cloud Firestore log("\nInitializing Cloud Firestore with key " + service_key) cred = credentials.Certificate(service_key) firebase_admin.initialize_app(cred) db = firestore.client() # Add data to Firestore log("\nAdding data to firestore...") added = 0 for index, row in resources_df.iterrows(): links = Links(row["android link"], row["card link"], row["facebook"], row["ios link"], row["website"]) resource = Resource(row["resource name"], True, row["description"], row["image link"], row["category"], row["tags"].split(", "), links) path = "resources" db.collection(path).add(resource.to_dict()) added += 1 log("\tAdded " + row["resource name"] + " to " + path) log("\nAdded " + str(added) + " entries to Firestore")