def get_urls_from_status_file(csvfilename): urls = [] headers, allurls = csv_to_array(csvfilename); for url in allurls: if url[1] != "dead": urls += [url[0]]; return urls
def create_deadalive_list():
#Get deadlist from file
fin = open("../../data/ush_deadurls.txt", "r")
deadurls = json.load(fin)
#Recreate fulllist from inputs
headers, urllist = csv_to_array(
"../../data/2014-11-18_Ushahidi_Site_Lists.csv")
crowdmaps, standalones = clean_ush_list(urllist)
#create array of dead, live, crowdmap from lists
liveurls = list(set(standalones).difference(set(deadurls)))
#Output results to csv file
fout = open("../../data/all_ush_status.csv", "wb")
csvout = csv.writer(fout, quoting=csv.QUOTE_NONNUMERIC)
csvout.writerow(["Site URL", "Status"])
for entry in liveurls:
csvout.writerow([entry, "live"])
for entry in crowdmaps:
csvout.writerow([entry, "crowdmap"])
for entry in deadurls:
csvout.writerow([entry, "dead"])
fout.close()
#Ideas for more: add country, #reports, lastreportyear, category, tech,
#livedead and scrape the description from the site. Add language? Add
#original description text (but translate main one into English?).
return ()
def get_urls_from_status_file(csvfilename):
urls = []
headers, allurls = csv_to_array(csvfilename)
for url in allurls:
if url[1] != "dead":
urls += [url[0]]
return urls
def clean_ush_url_list():
directoryurl = "http://tracker.ushahidi.com/"
headers, urllist = csv_to_array("../../data/brian_ush_lists.csv");
print("Original list length is "+str(len(urllist)));
crowdmaps, standalones = clean_ush_list(urllist);
print(str(len(crowdmaps))+ " crowdmaps, "+str(len(standalones))+" standalone instances");
hostcounts, deadlist = all_the_ush.count_hosts(standalones);
dict_to_csv("ush_hostcounts.csv", ["Host","Count"], hostcounts);
json.dump(deadlist, open("ush_deadurls.txt",'w')); #Write deadlist to file
def clean_ush_url_list():
directoryurl = "http://tracker.ushahidi.com/"
headers, urllist = csv_to_array("../../data/brian_ush_lists.csv")
print("Original list length is " + str(len(urllist)))
crowdmaps, standalones = clean_ush_list(urllist)
print(
str(len(crowdmaps)) + " crowdmaps, " + str(len(standalones)) +
" standalone instances")
hostcounts, deadlist = all_the_ush.count_hosts(standalones)
dict_to_csv("ush_hostcounts.csv", ["Host", "Count"], hostcounts)
json.dump(deadlist, open("ush_deadurls.txt", 'w'))
def correct_locations():
mapurl = "https://worldushahidis.crowdmap.com/";
[username, password] = get_ush_login("../secrets.csv", 'crowdmap')
headers, csvdata = csv_to_array('worldushahidislocated.csv')
for item in csvdata:
reportid = item[0]
print(reportid)
viewpayload = ushapy.get_ush_report(mapurl, reportid)
editpayload = ushapy.reformat_ush_api_report_view_to_edit(viewpayload)
editpayload['location_name'] = item[4]
editpayload['latitude'] = item[7]
editpayload['longitude'] = item[8]
#FIXIT: Also need to add 'incident_category' to the payload
resp = ushapy.edit_ush_report(mapurl, editpayload, username, password)
print(resp.text)
return()
def correct_locations():
mapurl = "https://worldushahidis.crowdmap.com/"
[username, password] = get_ush_login("../secrets.csv", 'crowdmap')
headers, csvdata = csv_to_array('worldushahidislocated.csv')
for item in csvdata:
reportid = item[0]
print(reportid)
viewpayload = ushapy.get_ush_report(mapurl, reportid)
editpayload = ushapy.reformat_ush_api_report_view_to_edit(viewpayload)
editpayload['location_name'] = item[4]
editpayload['latitude'] = item[7]
editpayload['longitude'] = item[8]
#FIXIT: Also need to add 'incident_category' to the payload
resp = ushapy.edit_ush_report(mapurl, editpayload, username, password)
print(resp.text)
return ()
def add_locations_to_ushcsv(filename):
countrylatlons = countrylocations.read_countrylatlons();
outrows = []
headers, inrows = csv_to_array(filename);
for row in inrows:
locname = row[4];
if locname == "unknown":
siteurl = row[2];
try:
row[10] = siteurl;
countryname, countrylat, countrylon = get_latlon_country(siteurl);
print(countryname+": "+str(countrylat)+","+str(countrylon))
row[4] = countryname;
row[14] = countryname;
row[7] = countrylat;
row[8] = countrylon;
if countryname != "unknown":
outrows += [row];
except:
print("failed for "+siteurl)
array_to_csv("worldushahidislocated.csv", outrows, headers);
return()
def create_deadalive_list():
#Get deadlist from file
fin = open("../../data/ush_deadurls.txt", "r");
deadurls = json.load(fin);
#Recreate fulllist from inputs
headers, urllist = csv_to_array("../../data/2014-11-18_Ushahidi_Site_Lists.csv");
crowdmaps, standalones = clean_ush_list(urllist);
#create array of dead, live, crowdmap from lists
liveurls = list(set(standalones).difference(set(deadurls)));
#Output results to csv file
fout = open("../../data/all_ush_status.csv", "wb");
csvout = csv.writer(fout, quoting=csv.QUOTE_NONNUMERIC);
csvout.writerow(["Site URL", "Status"]);
for entry in liveurls:
csvout.writerow([entry, "live"])
for entry in crowdmaps:
csvout.writerow([entry, "crowdmap"])
for entry in deadurls:
csvout.writerow([entry, "dead"])
fout.close();
#Ideas for more: add country, #reports, lastreportyear, category, tech,
#livedead and scrape the description from the site. Add language? Add
#original description text (but translate main one into English?).
return()
def add_locations_to_ushcsv(filename):
countrylatlons = countrylocations.read_countrylatlons()
outrows = []
headers, inrows = csv_to_array(filename)
for row in inrows:
locname = row[4]
if locname == "unknown":
siteurl = row[2]
try:
row[10] = siteurl
countryname, countrylat, countrylon = get_latlon_country(
siteurl)
print(countryname + ": " + str(countrylat) + "," +
str(countrylon))
row[4] = countryname
row[14] = countryname
row[7] = countrylat
row[8] = countrylon
if countryname != "unknown":
outrows += [row]
except:
print("failed for " + siteurl)
array_to_csv("worldushahidislocated.csv", outrows, headers)
return ()