def api_down_vote():
post_id = request.args.get('id', '')
if post_id is None:
return http_error('No post id in args', 400)
dal = Dal()
dal.down_vote(post_id)
return http_error('success', 200)
def get_post(post_id):
dal = Dal()
r = dal.get(post_id)
if r is not None:
response = {"date": r['date'], "post": r['post']}
return json.dumps(response), 200, {'ContentType': 'application/json'}
else:
return http_error('Not found', 400)
def searchHost(): print "Enter IP to Search" query = str(raw_input(prompt)) dal = Dal() hostresults = dal.searchHosts(query) if hostresults == None: print "Nothing Found..." else: print hostresults
def searchNetwork(): print "Enter CIDR to Search" cidr = str(raw_input(prompt)) dal = Dal() searchresults = dal.searchNetworks(cidr) if searchresults == None: print "Nothing Found..." else: print searchresults print cidr in dal.networks
def __init__(self):
# een attribuut van de klasse dal, die alle database acties regelt
self.dal = Dal()
# Het onderdeel voor het inloggen
self.username = input("Wat is je gebruikersnaam?")
login = self.user_object = self.dal.login_user(self.username)
# Checkt of de login goed of fout is gegaan.
if login != False:
self.menu()
else:
Program()
def import_job():
base_dir = os.path.dirname(os.path.realpath(__file__)) + '/'
data_dir = base_dir + 'data/'
dal = Dal(data_dir)
importer = Importer(data_dir, dal)
print('importing ' + str(datetime.now()))
importer.import_date(datetime.now())
def initialize_dal(self, mongo_host, mongo_port, mongo_pool_size, log_dir):
'''
Method to initialize the Data Access layer for handling all database
related operations.
:param mongo_host: host name for mongodb
:param mongo_port: port for mongodb
:param mongo_pool_size: connection pool size for mongodb client
:param logger: logger object for logging
:return:
'''
try:
self.dal = Dal(mongo_host, mongo_port, mongo_pool_size, self,
log_dir)
self.log_info("data access layer initialized successfully")
return True
except Exception as e:
self.log_error("DB init Error : {0}".format(e.message))
return False
def addNetwork(): print "Enter Network" cidr = str(raw_input(prompt)) print "Enter Network Note/Info" note = str(raw_input(prompt)) net = Network(cidr, note) dal = Dal() dal.addNetwork(net) dal.save()
def addHost(): print "Enter Host Ipv4 Address" ipadr = str(raw_input(prompt)) print "Enter Comment/Note" note = str(raw_input(prompt)) ip4 = buildIp(ipadr) host = Host(ip4, note) dal = Dal() dal.addHost(host) dal.save()
class Program:
######################## We hebben de todo's er bewust ingelaten, om te laten zien dat
######################## we er bewust van zijn dat nog niet alles 100% waterdicht werkt, hier hebben we helaas
######################## niet genoeg tijd voor gehad.
######################## We hebben ervoor gekozen om de class product zoveel mogelijk
######################## correct te maken met de dal class (zoals besproken met Miel).
# TODO: kijken of er meerdere classes gebruikt kunnen worden vanuit de dal class
# TODO: automatisch orders plaatsen afwerken
# TODO: de gebruiker altijd de optie geven om de applicatie af te sluiten
# TODO: terug naar het desbetreffende menu geleid worden naar een actie,
# en niet altijd het algemene menu.
def __init__(self):
# een attribuut van de klasse dal, die alle database acties regelt
self.dal = Dal()
# Het onderdeel voor het inloggen
self.username = input("Wat is je gebruikersnaam?")
login = self.user_object = self.dal.login_user(self.username)
# Checkt of de login goed of fout is gegaan.
if login != False:
self.menu()
else:
Program()
def menu(self):
# Via de flag kan het menu oneindig worden herhaald
flag = True
while flag:
print(
"Je bevindt je in het menu. Wat wil je doen?\n\n"
"Kies 'u' om naar de opties voor gebruikers te gaan\n"
"Kies 'l' om naar de opties voor leveranciers te gaan\n"
"Kies 'o' om naar de opties van orders te gaan\n"
"Kies 'p' om naar de opties voor producten te gaan\n"
"Kies 'lo' om uit te loggen \n"
"Kies 'e' om af te sluiten\n"
"Kies 'test' voor een demo van het automatische bevoorraadsysteem"
)
choice = input().lower()
# region User options
if choice == 'u':
print(
"****Welkom in het overzichtmenu voor gebruikersgegevens.**** \n"
"Kies 'i' om je gebruikersinfo te bekijken.\n"
"Kies 'all' om een lijst van alle users te zien\n"
"Kies 'a' om een user toe te voegen\n"
"Kies 'd' om een user te verwijderen\n"
"Kies 'c' om een user aan te passen\n"
"Kies 'm' om terug naar het menu te gaan \n"
"Kies 'e' om af te sluiten\n"
)
choice = input()
if choice == 'i':
self.user_object.get_info()
## lijst van alle users kunnen bekijken
elif choice == 'all':
self.dal.select_a_user('view')
## Het aamaken van een nieuwe gebruiker
# TODO: dubbelen controleren alvorens de nieuwe user aan te maken
elif choice == 'a':
if self.user_object.rol == "admin":
print("Je hebt ervoor gekozen om een nieuwe gebruiker aan te maken.")
flag2 = True
while flag2:
new_user_username = input ("Wat wordt de gebruikersnaam van de nieuwe gebruiker?")
if new_user_username == "" or new_user_username.isdigit():
print("Je input klopt niet helemaal, je hebt een lege "
"waarde of getal ingevoerd. Probeer het nog een keer.")
else:
flag2 = False
new_user_role = input("Is het een admin(typ a) of employee(typ e)?")
if new_user_role == 'a':
new_user_role = 'admin'
elif new_user_role == 'e':
new_user_role = 'employee'
self.dal.add_user(new_user_username,new_user_role)
input("\nDe nieuwe user " + new_user_username + " is toegevoegd!\n "
"Druk op enter om terug naar het menu te gaan")
pass
else:
print("Je hebt niet de rechten om deze actie uit te voeren.")
pass
## Het verwijderen van een user
elif choice == 'd':
print("Je hebt ervoor gekozen om een gebruiker te verwijderen.")
if self.user_object.rol == "admin":
selected_to_delete = self.dal.select_a_user('action')
confirm_delete = input("Je hebt gekozen om " + selected_to_delete + " te verwijderen. Weet je dit zeker? (y/n)")
if confirm_delete == "y":
self.dal.delete_user(selected_to_delete)
input("Je hebt met succes " + selected_to_delete + " verwijderd.\n"
"Druk op enter om terug naar het menu te gaan.")
else:
pass
pass
else:
print("Je hebt niet de rechten om deze actie uit te voeren.\n"
"Druk op enter om terug naar het menu te gaan.")
pass
## Het kiezen en aanpassen van een user
elif choice == 'c':
print("Je hebt ervoor gekozen om de gegevens van een gebruiker aan te passen.")
if self.user_object.rol == "admin":
selected_to_change = self.dal.select_a_user('change')
new_user_username = input("Wat wordt de nieuwe gebruikersnaam? "
"\n(druk op enter om de huidige gebruikersnaam te behouden)")
if new_user_username == "" or new_user_username.isdigit():
print("De oude username " + selected_to_change.username + " wordt behouden")
new_user_username = selected_to_change.username
new_user_role = input("Is het een admin(a) of employee(e)?"
"\n(druk op enter om de huidige rol te behouden)")
if new_user_role == 'a':
new_user_role = 'admin'
elif new_user_role == 'e':
new_user_role = 'employee'
else:
print("Je hebt een foutieve waarde ingevoerd. Rol blijft onveranderd")
new_user_role = selected_to_change.rol
self.dal.modify_user(selected_to_change.userid, new_user_username, new_user_role)
input("De gegevens van " + new_user_username + " zijn succesvol aangepast. \n"
"Druk op enter om terug naar het menu te gaan.")
pass
else:
print("Je hebt niet de rechten om deze actie uit te voeren."
"\n Druk op enter om terug naar het menu te gaan.")
pass
elif choice == 'm':
self.menu()
elif choice == 'e':
exit()
pass
# endregion
# region Leverancier options
elif choice == 'l':
print(
"****Welkom in het overzichtmenu voor gebruikersgegevens.**** \n"
"Kies 'a' om een leverancier toe te voegen\n"
"Kies 'd' om een leverancier te verwijderen\n"
"Kies 'c' om een leverancier aan te passen\n"
"Kies 'all voor een volledige lijst van leveranciers\n"
"Kies 'm' om terug naar het menu te gaan \n"
"Kies 'e' om af te sluiten\n"
)
choice = input()
if choice == 'a':
print("Je hebt ervoor gekozen om een nieuwe leverancier aan te maken.")
flag2 = True
while flag2:
new_company_name = input("Wat is de naam van de leverancier?")
if new_company_name == "":
print("Je hebt een lege waarde ingevuld, probeer het nog een keer")
else:
flag2 = False
flag2 = True
while flag2:
new_company_adres = input("Wat is het adres van de leverancier?")
if new_company_adres == "" or new_company_adres.isdigit():
print("Je hebt een lege waarde of getal ingevuld, probeer het nog een keer")
else:
flag2 = False
flag2 = True
while flag2:
new_company_email = input("Wat is het emailadres van de leverancier?")
if new_company_email == "" or new_company_email.isdigit():
print("Je hebt een lege waarde of getal ingevuld, probeer het nog een keer")
else:
flag2 = False
flag2 = True
while flag2:
new_company_levertijd = input("Wat is de levertijd van de leverancier?")
if new_company_levertijd == "" or new_company_levertijd.isdigit():
print("Je hebt een lege waarde of getal ingevuld, probeer het nog een keer")
else:
flag2 = False
self.dal.add_company(new_company_name, new_company_adres, new_company_email, new_company_levertijd)
input("\nDe nieuwe leverancier " + new_company_name + " is toegevoegd!\n"
"Druk op enter om terug naar het menu te gaan.")
pass
elif choice == 'd':
print("Je hebt ervoor gekozen om een leverancier te verwijderen.")
selected_to_delete = self.dal.select_a_company('action')
confirm_delete = input(
"Je hebt gekozen om " + selected_to_delete + " te verwijderen. Weet je dit zeker? (y/n)")
if confirm_delete == "y":
self.dal.delete_company(selected_to_delete)
input(selected_to_delete + " is verwijderd.\n"
"Druk op enter om terug naar het menu te gaan.")
else:
pass
pass
elif choice == 'c':
print("Je hebt ervoor gekozen om een leverancier aan te passen.")
selected_to_change = self.dal.select_a_company('change')
new_company_name = input("Wat is de nieuwe naam van de leverancier? "
"\n(druk op enter om de huidige naam te behouden)")
if new_company_name == "":
print("De leveranciernaam wordt niet aangepast.")
new_company_name = selected_to_change.leveranciernaam
new_company_adres = input("Wat is het adres van de leverancier?"
"\n(druk op enter om het huidige adres te behouden)")
if new_company_adres == "" or new_company_adres.isdigit():
print("het adres wordt niet aangepast.")
new_company_adres = selected_to_change.leverancieradres
new_company_email = input("Wat is het emailadres van de leverancier?"
"\n(druk op enter om het huidige email adres te behouden)")
if new_company_email == "" or new_company_email.isdigit():
print("Het email adres wordt niet aangepast.")
new_company_email = selected_to_change.leverancieremail
new_company_levertijd = input("Wat is de levertijd van de leverancier?"
"\n(druk op enter om de huidige levertijd te behouden)")
if new_company_levertijd == "" or new_company_levertijd.isdigit():
print("De levertijd van het bedrijf wordt niet aangepast.")
new_company_levertijd = selected_to_change.levertijd
self.dal.modify_company(selected_to_change.leverancierID, new_company_name, new_company_adres, new_company_email, new_company_levertijd)
input("De gegevens van " + new_company_name + " zijn succesvol aangepast.\n"
"Druk op enter om terug naar het menu te gaan.")
pass
elif choice == 'all':
self.dal.select_a_company('show')
input("Druk op enter om terug naar het menu te gaan.")
pass
elif choice == 'm':
self.menu()
elif choice == 'e':
exit()
pass
#endregion
# region Order options
elif choice == 'o':
print(
"****Welkom in het overzichtmenu voor orders.**** \n"
"Kies 'all' om alle orders te bekijken\n"
"Kies 'a' om een order handmatig te plaatsen\n"
"Kies 'c' om een order aan te passen\n"
"Kies 'd' om een order te annuleren\n"
"Kies 'm' om terug naar het menu te gaan \n"
"Kies 'e' om af te sluiten\n"
)
choice = input()
if choice == 'all':
self.dal.select_an_order("view")
input("Druk op enter om terug naar het menu te gaan.")
pass
if choice == 'a':
print("Je hebt ervoor gekozen om een order te plaatsen.\n"
"Kies eerst uit welke producten je wil bestellen.")
items_to_order = self.dal.select_a_product("placeorder")
new_order_id = self.dal.prepare_order()
for product, quantity in items_to_order.items():
self.dal.place_order(new_order_id, product, quantity)
whole_product = self.dal.get_product_by_id(product)
self.dal.modify_product(whole_product.productID,whole_product.leverancierID,
whole_product.productnaam, whole_product.inkoopprijs,
int(whole_product.voorraad) + int(quantity),
whole_product.min,whole_product.max)
input("De bestelling is succesvol geplaatst! Druk op enter om door te gaan.")
pass
elif choice == 'c':
## Het aanpassen van de status
print("Je hebt gekozen om een order aan te passen.")
flag3 = True
while flag3:
try:
selected_to_change = self.dal.select_an_order('action')
what_to_change = ""
while what_to_change != "1" and what_to_change != "2":
what_to_change = input(str("""Wat wil je aanpassen aan deze order?
[1] Status
[2] Producten
> """))
if what_to_change == "1":
print("'1.' In behandeling\n"
"'2.' Betaald\n"
"'3.' Verzonden\n"
"'4.' Geleverd")
flag2 = True
while flag2:
new_order_status = input("Wat is de nieuwe status? (nummer)")
if int(new_order_status) == 1:
new_order_status = "In behandeling"
flag2 = False
elif int(new_order_status) == 2:
new_order_status = "Betaald"
flag2 = False
elif int(new_order_status) == 3:
new_order_status = "Verzonden"
flag2 = False
elif int(new_order_status) == 4:
new_order_status = "Geleverd"
flag2 = False
else:
print("Je invoer klopt niet, probeer het nog een keer")
self.dal.modify_order(selected_to_change, "status", new_order_status)
input("De nieuwe status is met succes aangepast. "
"Druk op enter om terug naar het menu te gaan.")
flag3 = False
## het aanpassen van de producten van een order
#TODO: optie om extra producten toe te voegen zolang order nog in behandeling is
elif what_to_change == "2":
print("Je hebt ervoor gekozen om de producten binnen de bestelling aan te passen.\n"
"De volgende producten zitten binnen de order.")
chosen_product_and_order = self.dal.select_current_order_products("action", selected_to_change)
choice = input("Wat wil je met dit product doen?\n"
"Kies 'v' om te verwijderen\n"
"Kies 'h' om de hoeveelheid aan te passen")
if choice == "v":
for order, product in chosen_product_and_order.items():
self.dal.delete_order_product(order, product)
input("Het product is succesvol uit het order verwijderd.\n"
"Druk op enter om door te gaan.")
flag3 = False
pass
elif choice == "h":
flag2 = True
while flag2:
for order,product in chosen_product_and_order.items():
whole_product = self.dal.get_product_by_id(product)
max_to_order = int(whole_product.max) - int(whole_product.voorraad)
nieuwe_hoeveelheid = input("Naar welke hoeveelheid moet het aangepast worden? "
"(max " + str(max_to_order) + " )")
if nieuwe_hoeveelheid == "" or nieuwe_hoeveelheid.isalpha() \
or int(nieuwe_hoeveelheid) > max_to_order or int(nieuwe_hoeveelheid) == 0:
print("Je input is geen cijfer of te hoog. Probeer het nog een keer.")
else:
self.dal.modify_order(chosen_product_and_order, "hoeveelheid", nieuwe_hoeveelheid)
input("Het product is met succes aangepast. \n"
"Druk op enter om terug naar het menu te gaan")
flag2 = False
flag3 = False
pass
pass
except ValueError:
print("Je input klopt niet helemaal. Probeer het nog een keer.")
flag3 = True
elif choice == 'd':
# TODO: Maken dat alleen bestellingen in behandeling verwijderd kunnen worden
chosen_order_to_delete = self.dal.select_an_order("action")
self.dal.delete_order(chosen_order_to_delete)
input("De order is succesvol verwijderd.\n"
"Druk op enter om door te gaan.")
pass
elif choice == 'm':
self.menu()
elif choice == 'e':
exit()
pass
#endregion
#region Product options
elif choice == 'p':
print(
"Kies 'all' om alle producten te bekijken\n"
"Kies 'a' om een product toe te voegen\n"
"Kies 'c' om een product aan te passen\n"
"Kies 'd' om een product te verwijderen\n"
"Kies 'm' om terug naar het menu te gaan \n"
"Kies 'e' om af te sluiten\n"
)
choice = input()
if choice == 'all':
self.dal.select_a_product("view")
pass
elif choice == 'a':
new_product_name = input("Wat is de naam van het nieuwe product?")
while new_product_name == "":
new_product_name = input("Geen naam ingevoerd, probeer het opnieuw:")
print("Kies uit de lijst met leveranciers wie het product levert")
new_product_leverancier = self.dal.select_a_company("addproduct")
new_product_leverancier = new_product_leverancier[0]
print(new_product_leverancier)
new_product_inkoopprijs = input("Wat is de inkoopprijs? (bijv 1.50)")
while float(new_product_inkoopprijs) <= 0 or new_product_inkoopprijs == \
"" or new_product_inkoopprijs.isalpha():
new_product_inkoopprijs = input("Je ingevoerde inkoopprijs "
"klopt niet helemaal. Probeer het nog een keer. ")
new_product_voorraad = 0
new_product_minvoorraad = input("Geef een limiet van de minimum voorraad")
while int(new_product_minvoorraad) < 0 or new_product_minvoorraad == \
"" or new_product_minvoorraad.isalpha():
new_product_minvoorraad = input("Je ingevoerde minimum klopt niet "
"helemaal. Probeer het nog een keer. ")
new_product_maxvoorraad = input("Geef een limiet van de maximum voorraad")
while int(new_product_maxvoorraad) < int(new_product_minvoorraad) or new_product_maxvoorraad == ""\
or new_product_maxvoorraad.isalpha():
new_product_maxvoorraad = input(f"Maximum voorraad kan niet kleiner zijn dan "
f"minimum voorraad ({new_product_minvoorraad}):,"
f"of je hebt een foute value ingevuld.")
self.dal.add_product(new_product_leverancier, new_product_name, new_product_inkoopprijs,
new_product_voorraad, new_product_minvoorraad, new_product_maxvoorraad)
input("\nHet nieuwe product " + new_product_name + " is toegevoegd!"
"Druk op enter om door te gaan.")
pass
elif choice == 'c':
print("Je hebt ervoor gekozen om een product aan te passen.")
selected_to_change = self.dal.select_a_product('productaanpassen')
leverancier = self.dal.select_a_company("changeproduct")
modified_values = selected_to_change.modify_product(leverancier[0])
self.dal.modify_product(int(selected_to_change.productID), int(modified_values[0]), str(modified_values[1]),
float(modified_values[2]), int(modified_values[3]), int(modified_values[4]),
int(modified_values[5]))
if int(modified_values[3]) < int(modified_values[4]):
print("De voorraad is onder het minumum, een nieuwe order wordt aangemaakt")
outcome_check = int(modified_values[5]) - int(modified_values[3])
new_order_id = self.dal.prepare_order()
self.dal.place_order(new_order_id, int(selected_to_change.productID), outcome_check)
self.dal.modify_product(int(selected_to_change.productID), int(modified_values[0]), str(modified_values[1]),
float(modified_values[2]), int(modified_values[5]), int(modified_values[4]),
int(modified_values[5]))
input(modified_values[1] + " is succesvol aangepast.\n"
"Druk op enter om verder te gaan.")
pass
elif choice == 'd':
selected_to_delete = self.dal.select_a_product('delete')
confirm_delete = input(
"Je hebt gekozen om " + str(selected_to_delete[1]) + " te verwijderen. Weet je dit zeker? (y/n)")
if confirm_delete == "y":
self.dal.delete_product(selected_to_delete[0])
input(selected_to_delete[1] + " succesvol verwijderd. \n"
"Druk op enter om terug naar het menu te gaan.")
else:
pass
pass
elif choice == 'm':
self.menu()
elif choice == 'e':
exit()
pass
#endregion
elif choice == 'lo':
Program()
elif choice == 'e':
exit()
############# Test voor het automatisch bestellen van een product
# TODO: moet nog beter uitgewerkt worden
elif choice == 'test':
self.dal.check_all_products()
class Utility:
'''
Utility class for processing and restructuring data.
'''
# logger object to log progress and application status
logger = None
# data access layer object for handling all data access operations
dal = None
def _init_(self):
return
def load_properties(self, constants):
'''
A method to load the property file. If the property file is missing or
any exception occurs,
then a default set of properties are loaded and returned.
:return: Constant c that has all the property values that are shared across
the application
'''
p = Properties()
c = constants
try:
with open(c.prop_file, "rb") as f:
p.load(f)
if p.__contains__("url"):
c.url = p["url"].data
if p.__contains__("url2"):
c.mongo_pool_size = p["url2"].data
if p.__contains__("working_dir"):
c.working_dir = p["working_dir"].data
if p.__contains__("download_limit"):
c.download_limit = int(p["download_limit"].data)
if p.__contains__("download_sort"):
c.download_sort = bool(p["download_sort"].data)
if p.__contains__("log_dir"):
c.log_dir = p["log_dir"].data
if p.__contains__("mongo_host"):
c.mongo_host = p["mongo_host"].data
if p.__contains__("mongo_port"):
c.mongo_port = int(p["mongo_port"].data)
if p.__contains__("db_name"):
c.db_name = p["db_name"].data
if p.__contains__("mongo_pool_size"):
c.mongo_pool_size = int(p["mongo_pool_size"].data)
return c
except IOError as e:
print("I/O error({0}): {1} ;reason {2}".format(
e.errno, e.strerror, e.message))
print(
"error while loading the property file. Returning default properties "
"from Constants.")
return c
except Exception as e:
print "Unexpected error:", sys.exc_info()[0]
print(
"Some unexpected error occured while loading the property file. "
"Returning default properties from Constants.")
return c
def initialize_logger(self, log_path):
'''
Method to initialize the logger for logging application progress and events
:param log_path: path for storing the app logs
:return:
'''
try:
# logging.basicConfig(filename=log_path,format='%(asctime)s %(
# message)s %(
# levelname)s', datefmt='%m/%d/%Y %I:%M:%S %p', level=logging.DEBUG)
self.logger = logging.getLogger("utils")
self.logger.setLevel(logging.DEBUG)
# create a file handler
log_path = os.path.join(log_path, "app.utils.log")
handler = logging.handlers.RotatingFileHandler(log_path,
maxBytes=5000,
backupCount=5)
# create a logging format
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler.setFormatter(formatter)
# add the handlers to the logger
self.logger.addHandler(handler)
except Exception as e:
print(e.__str__())
print(
"unable to set specified logpath. Setting logpath to default which is "
"app.log in the application folder itself.")
# logging.basicConfig(filename="app.log", format='%(asctime)s %(
# message)s %(
# levelname)s', datefmt='%m/%d/%Y %I:%M:%S %p', level=logging.DEBUG)
self.logger = logging.getLogger("utils")
self.logger.setLevel(logging.DEBUG)
# create a file handler
handler = logging.handlers.RotatingFileHandler("app.utils.log",
maxBytes=5000,
backupCount=5)
# create a logging format
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler.setFormatter(formatter)
# add the handlers to the logger
self.logger.addHandler(handler)
def parse_cmdline_args(self, argv, constants):
'''
Method to parse command line arguments and save them in the constants class
:param argsv: list of command line arguments
:return: constants object that holds all the properties that is shared
across the application
'''
try:
opts, args = getopt.getopt(argv, "hi:o:", ["pfile=", "pfile="])
except getopt.GetoptError:
print 'app.py -pfile <property file name> -lfile <log file name>'
sys.exit(2)
for opt, arg in opts:
if opt == '-h':
print 'app.py -pfile <property file> -lfile <log file>'
sys.exit()
elif opt in ("-p", "--pfile"):
if arg is not None or not arg:
constants.prop_file = arg
elif opt in ("-l", "--lfile"):
if arg is not None or not arg:
constants.log_dir = arg
return constants
def create_cache_file(self, file_path, content):
try:
with open(file_path, "w") as text_file:
text_file.write(content)
return True
except Exception as e:
self.log_error(
"Exception occurred while writing the cache file to the working "
"directory.\n" + "Error message: " + e.message)
return False
def get_cached_file_urls(self, file_path):
if os.path.isfile(file_path):
try:
with open(file_path, 'r') as content_file:
content = content_file.read()
return content
except Exception as e:
self.log_error(
"Exception occurred while reading the file url cache file. "
"Error Message : " + e.message +
"\n returning None to fetch the file list from the given url"
)
return None
else:
return None
def get_file_urls(self, url):
'''
Method that makes an HTTP GET request to the given url and extracts all the
current file names from GDELT2.0
master file list and converts it into a dictionary
:param url: url for fetching GDELT2.0 master files list
:return: structured files list (grouped categorically and chronologically)
'''
try:
str_current_files = urllib2.urlopen(url).read()
list_current_files = str_current_files.splitlines()
print("number of lines in the file : " +
str(len(list_current_files)))
grouped_current_files = self.chunks(list_current_files, 3)
grouped_current_files_dict = []
for i, x in enumerate(grouped_current_files):
if len(x[0].split()) < 3 or len(x[1].split()) < 3 or len(
x[2].split()) < 3:
print(i)
continue
fileobj = {
"event": {
"id": x[0].split()[0],
"chksum": x[0].split()[1],
"url": x[0].split()[2],
"filename": x[0].split()[-1].split("/")[-1],
"status": 0
},
"mentions": {
"id": x[1].split()[0],
"chksum": x[1].split()[1],
"url": x[1].split()[2],
"filename": x[1].split()[-1].split("/")[-1],
"status": 0
},
"kgraph": {
"id": x[2].split()[0],
"chksum": x[2].split()[1],
"url": x[2].split()[2],
"filename": x[2].split()[-1].split("/")[-1],
"status": 0
},
"timestamp":
int(x[0].split()[-1].split("/")[-1].split(".")[0])
}
grouped_current_files_dict.append(fileobj)
return grouped_current_files_dict
except Exception as e:
self.log_error("URL Error : {0}".format(e.message))
self.log_error(
"Failed to open url. Please provide a valid url for fetching the "
"files.")
return None
def initialize_dal(self, mongo_host, mongo_port, mongo_pool_size, log_dir):
'''
Method to initialize the Data Access layer for handling all database
related operations.
:param mongo_host: host name for mongodb
:param mongo_port: port for mongodb
:param mongo_pool_size: connection pool size for mongodb client
:param logger: logger object for logging
:return:
'''
try:
self.dal = Dal(mongo_host, mongo_port, mongo_pool_size, self,
log_dir)
self.log_info("data access layer initialized successfully")
return True
except Exception as e:
self.log_error("DB init Error : {0}".format(e.message))
return False
def dump_file_urls(self, current_file_url_list, db_name, logger=None):
'''
Utility method to dump newly added file urls to db
:param current_file_url_list: new list of files
:param db_name: name of the database that we will query
:param logger: logger object
:return: status of insert
'''
self.log_info("dumping newly added files to db")
dump_status = self.dal.insert_new_files(current_file_url_list, db_name,
self.logger)
return dump_status
def download_and_insert_pending_files(self,
db_name,
working_dir,
event_import_command,
mentions_import_command,
download_limit=0,
download_sort=False):
'''
Method that downloads all the files with status 0 and inserts them into the
events and mentions collection
:param db_name: name of the db
:return: insert status
'''
# Get all the pending files from the database
pending_files = self.dal.get_file_urls_by_status(db_name,
working_dir,
status=0,
sort=download_sort)
# Check if there are no pending files
if pending_files.count() is 0:
return 0, 0 # there are no pending files in db
# set counter to check the number of inserts later
insert_count = 0
if download_limit == 0:
download_limit = len(pending_files)
# Iterate through the pending files
for n, fileobj in enumerate(pending_files):
if n == download_limit:
break
self.log_info("downloading file " + str(n) + " of " +
str(pending_files.count()) + " files")
# Url for event files
events_url = fileobj["event"]["url"]
# download and insert status of event file
events_status = fileobj["event"]["status"]
# Url of the mentions file
mentions_url = fileobj["mentions"]["url"]
# status of the mentions file
mentions_status = fileobj["mentions"]["status"]
# skip download if the status is 1
if events_status is 1:
e_dwn_status = True
else:
e_dwn_status, events_comp_file_path = self.download_file(
working_dir, events_url)
if mentions_status is 1:
m_dwn_status = True
else:
m_dwn_status, mentions_comp_file_path = self.download_file(
working_dir, mentions_url)
self.log_info("Successfully downloaded file " + str(n) + " of " +
str(pending_files.count()) + " files")
if e_dwn_status and m_dwn_status:
self.log_info("unzipping files \n" + events_comp_file_path +
"\n and \n" + mentions_comp_file_path)
# unzip the files in the working directory
events_file_path = self.unzip_file(events_comp_file_path,
working_dir)
mentions_file_path = self.unzip_file(mentions_comp_file_path,
working_dir)
self.log_info("Successfully unzipped files \n" +
events_comp_file_path + "\n "
"and"
" \n" + mentions_comp_file_path + " to " +
events_file_path + "\nand\n" +
mentions_file_path)
# Check if the files were successfully unzipped and if not then
# skip the files
if events_file_path is None or mentions_file_path is None:
self.log_error(
"Skipping file " + events_url + " and " +
mentions_url +
" because one of them could not be unzipped.")
continue
self.log_info("Importing files \n" + events_file_path +
"\n and "
"\n" + mentions_file_path +
"\n into database: " + db_name)
# import the events csv file into the database
import_events_status = self.import_csv_to_db(
events_file_path, db_name, "cameo_events",
event_import_command)
# Check if import was successfull. If not then log error but if
# successfull the update
# file status and delete the file from the working directory
if import_events_status is True:
self.log_info("Successfully imported files \n" +
events_file_path + "\n into database: " +
db_name)
self.log_info("Updating status of file \n" +
events_file_path + "\n in database: " +
db_name + " to 1 (downloaded "
"and imported)")
# Update the file status to 1
e_update_status = self.dal.update_file_status(db_name,
fileobj,
"event",
status=1)
# Check whether file status update was successfull or not.
# If success then delete the files from the working directory.
if e_update_status is True:
self.log_info(
"Successfully updated status of file \n" +
events_file_path + "\n in database: " + db_name +
" to 1 (downloaded and "
"imported)")
# Delete the files from the working directory
self.delete_files(
[events_comp_file_path, events_file_path])
else:
self.log_error("Could not update status of file \n" +
events_file_path + "\n in database: " +
db_name + " to 1 (downloaded and "
"imported)")
# Import mentions csv into the database
import_mentions_status = self.import_csv_to_db(
mentions_file_path, db_name, "cameo_mentions",
mentions_import_command)
# Check if import was successful. If not then log error but if
# successful the update
# file status and delete the file from the working directory
if import_mentions_status is True:
self.log_info("Successfully imported files \n" +
mentions_file_path + "\n "
"into database: " + db_name)
# update the file ststus to 1
m_update_status = self.dal.update_file_status(db_name,
fileobj,
"mentions",
status=1)
# Check if update was successful or not.
# If it was the delete the files from the working directory
if m_update_status is True:
self.log_info(
"Successfully updated status of file \n" +
events_file_path + "\n in database: " + db_name +
" to 1 (downloaded and "
"imported)")
# delete the files from the working directory
self.delete_files(
[mentions_comp_file_path, mentions_file_path])
else:
self.log_error("Could not update status of file \n" +
events_file_path + "\n in database: " +
db_name + " to 1 (downloaded and "
"imported)")
# increate the import counter if everything was fine
if import_events_status and import_mentions_status:
insert_count = insert_count + 1
else:
self.log_error("Skipping file " + events_url + " and " +
mentions_url +
" because of unsuccessful download.")
continue
# check if all the pending files were imported successfully and return
# a status
# accordingly
if insert_count == download_limit:
return 1, insert_count # all files were successfully downloaded and
# inserted
else:
return 2, insert_count # Not all files were downloaded and inserted
# successfully
def update_metadate(self, db_name):
'''
Method to update meta information about the entire database and new updates
:param db_name: name of the database
:return: update status
'''
update_status = self.dal.update_metadata(db_name)
return update_status
def update_global_impact_map(self,
db_name,
end_date,
start_date=0,
update=False):
'''
Method to update the data for the global impact map in the global dashboard
:param db_name: Name of the DB
:param end_date: End date for selection
:param start_date: start date for selection
:return: Status of update
'''
globa_impact_map_data = self.dal.get_global_impact_map_data(
db_name, end_date, start_date)
if globa_impact_map_data is None:
return False
map_update_status = self.dal.update_impact_map(db_name,
globa_impact_map_data,
update)
if map_update_status is True:
return True
else:
return False
def update_linked_locations(self, db_name, svg):
'''
Method to update all linked locations in the last one day
:param db_name:name of the db
:param svg: image for the map in svg string
:return: update status
'''
last_update_date_gdelt = self.dal.get_last_update_date(db_name)
if last_update_date_gdelt is None:
last_update_date_gdelt = self.get_present_date_time()
end_date = self.gdelt_date_to_datetime(last_update_date_gdelt)
start_date = self.datetime_to_gdelt_date(end_date -
dt.timedelta(minutes=15))
linked_locations = self.dal.get_linked_locations(
db_name, last_update_date_gdelt, svg, start_date=start_date)
if linked_locations is None:
return False
update_status = self.dal.update_linked_locations(
db_name, linked_locations)
return update_status
def update_event_count(self, db_name, end_date, start_date):
'''
:param db_name:
:param n_days:
:return:
'''
event_count = self.dal.get_event_count(db_name,
end_date,
start_date=start_date)
if event_count is None:
return False
update_status = self.dal.update_event_count(db_name, event_count)
return update_status
def update_globe_viz(self, db_name, end_date, start_date, update=False):
globe_viz = self.dal.get_globe_viz(db_name,
end_date,
start_date=start_date)
if globe_viz is None:
return False
data = []
for i, event in enumerate(globe_viz):
if i != 0 and i % 2000 == 0:
print(str(i) + ' events formatted.')
break
e_obj = {}
e_obj["type"] = 'Feature'
e_obj["geometry"] = {
"type": "Point",
"coordinates":
[event['ActionGeo_Long'], event['ActionGeo_Lat'], 0]
}
del event['ActionGeo_Long']
del event['_id']
del event['ActionGeo_Lat']
e_obj["properties"] = event
data.append(e_obj)
update_obj = {'features': data, 'type': 'FeatureCollection'}
#json_file_str = json.dumps(update_obj)
with open('./globe_viz/' + str(start_date) + '.json', 'w') as outfile:
json.dump(update_obj, outfile)
#update_status = self.dal.update_globe_viz(db_name, update_obj, update)
return True
def update_actor_network(self,
db_name,
end_date,
start_date,
update=False):
'''
:param db_name:
:param n_days:
:return:
'''
actor_network = self.dal.get_actor_network(db_name,
end_date,
start_date=start_date)
if actor_network is None:
return False
update_status = self.dal.update_actor_network(db_name, actor_network,
update)
return update_status
def update_mentions_timeline(self, db_name):
last_update_date_gdelt = self.dal.get_last_update_date(db_name)
if last_update_date_gdelt is None:
last_update_date_gdelt = self.get_present_date_time()
first_update_date_gdelt = self.dal.get_first_update_date_mt(db_name)
if first_update_date_gdelt is None:
first_update_date_gdelt = self.get_present_date_time(
) - dt.timedelta(weeks=20)
start_date = self.gdelt_date_to_datetime(first_update_date_gdelt)
start_date = start_date.replace(hour=0, minute=0, second=0)
end_date = self.gdelt_date_to_datetime(last_update_date_gdelt)
days_delta = (end_date - start_date).days
adder = 0
data = []
sources = []
for i in range(days_delta + 1):
temp_end_date = None
if adder == 0:
start_date = start_date + dt.timedelta(days=adder)
adder += 1
else:
start_date = start_date + dt.timedelta(days=adder)
start_date = start_date.replace(hour=0, minute=0, second=0)
if i == days_delta:
temp_end_date = start_date.replace(hour=end_date.hour,
minute=end_date.minute,
second=end_date.second)
else:
temp_end_date = start_date.replace(hour=23,
minute=59,
second=59)
print('Mentions Timeline :\nStart Date : ' + str(start_date) +
'\nEnd Date : ' + str(end_date))
articles_per_category = self.dal.get_articles_per_category(
db_name, self.datetime_to_gdelt_date(temp_end_date),
self.datetime_to_gdelt_date(start_date))
m_obj = {
"year": start_date.strftime("%d-%m-%Y"),
'citation_only': 0,
'web': 0
}
for cat in articles_per_category["data"]:
cat_name = cat["source"].replace(" ", "_")
n_mentions = cat["n_mentions"]
if cat_name not in sources: sources.append(cat_name)
m_obj[cat_name] = n_mentions
data.append(m_obj)
update_obj = {
"data": data,
"timestamp": self.get_present_date_time(),
"sources": sources
}
update_status = self.dal.update_mentions_timeline(db_name, update_obj)
return update_status
if update_status is True:
print("Successfully updated MT for : " + str(start_date))
else:
print("Failed to updated MT for : " + str(start_date))
def update_articles_per_category(self, db_name):
'''
articles_per_category:
:param db_name:
:return:
'''
end_date = self.dal.get_last_update_date(db_name)
if end_date is None:
end_date = self.get_present_date_time()
articles_per_category = self.dal.get_articles_per_category(
db_name, end_date, start_date=0)
if articles_per_category is None:
return False
update_status = self.dal.update_articles_per_category(
db_name, articles_per_category)
return update_status
def update_high_impact_events(self, db_name, limit=10):
last_update_date_gdelt = self.dal.get_last_update_date(db_name)
if last_update_date_gdelt is None:
last_update_date_gdelt = self.get_present_date_time()
end_date_15 = self.gdelt_date_to_datetime(last_update_date_gdelt)
start_date_15 = self.datetime_to_gdelt_date(end_date_15 -
dt.timedelta(minutes=15))
end_date_today = end_date_15
start_date_today = self.datetime_to_gdelt_date(
datetime(*(end_date_today.year, end_date_today.month,
end_date_today.day, 0, 0, 0)))
end_date_today = self.datetime_to_gdelt_date(end_date_today)
last_month, last_year = self.get_last_month_year(end_date_15)
days_in_last_month = monthrange(last_year, last_month)[1]
start_date_month = self.datetime_to_gdelt_date(
datetime(*(last_year, last_month, days_in_last_month, 0, 0, 0)))
end_date_month = end_date_today
h_i_events_15 = self.dal.get_high_impact_events(
db_name,
last_update_date_gdelt,
start_date=start_date_15,
limit=limit)
h_i_events_today = self.dal.get_high_impact_events(
db_name, end_date_today, start_date=start_date_today, limit=limit)
h_i_events_month = self.dal.get_high_impact_events(
db_name, end_date_month, start_date=start_date_month, limit=limit)
if h_i_events_15 is None or h_i_events_today is None or h_i_events_month is\
None:
return False
hir_object = {
"hie_15": h_i_events_15,
"hie_today": h_i_events_today,
"hie_month": h_i_events_month,
"timestamp": self.get_present_date_time()
}
update_status = self.dal.update_high_impact_events(db_name, hir_object)
return update_status
def update_high_impact_regions(self, db_name, limit=10):
last_update_date_gdelt = self.dal.get_last_update_date(db_name)
if last_update_date_gdelt is None:
last_update_date_gdelt = self.get_present_date_time()
end_date_15 = self.gdelt_date_to_datetime(last_update_date_gdelt)
start_date_15 = self.datetime_to_gdelt_date(end_date_15 -
dt.timedelta(minutes=15))
end_date_today = end_date_15
start_date_today = self.datetime_to_gdelt_date(
datetime(*(end_date_today.year, end_date_today.month,
end_date_today.day, 0, 0, 0)))
end_date_today = self.datetime_to_gdelt_date(end_date_today)
last_month, last_year = self.get_last_month_year(end_date_15)
days_in_last_month = monthrange(last_year, last_month)[1]
start_date_month = self.datetime_to_gdelt_date(
datetime(*(last_year, last_month, days_in_last_month, 0, 0, 0)))
end_date_month = end_date_today
h_i_regions_15 = self.dal.get_high_impact_regions(
db_name,
last_update_date_gdelt,
start_date=start_date_15,
limit=limit)
h_i_regions_today = self.dal.get_high_impact_regions(
db_name, end_date_today, start_date=start_date_today, limit=limit)
h_i_regions_month = self.dal.get_high_impact_regions(
db_name, end_date_month, start_date=start_date_month, limit=limit)
if h_i_regions_15 is None or h_i_regions_today is None or h_i_regions_month\
is None:
return False
hir_object = {
"hir_15": h_i_regions_15,
"hir_today": h_i_regions_today,
"hir_month": h_i_regions_month,
"timestamp": self.get_present_date_time()
}
update_status = self.dal.update_high_impact_regions(
db_name, hir_object)
return update_status
def update_overall_stats(self, db_name):
'''
Method to update overall events, mentions and countries statistics
:param db_name: name of the db
:return: status of update
'''
dt_now = self.dal.get_last_update_date(db_name)
dt_obj_now = self.gdelt_date_to_datetime(dt_now)
last_month, last_year = self.get_last_month_year(dt_obj_now)
days_in_last_month = monthrange(last_year, last_month)[1]
dt_last_month = self.datetime_to_gdelt_date(
datetime(*(last_year, last_month, days_in_last_month, 0, 0, 0)))
dt_today_start = self.datetime_to_gdelt_date(
datetime(*(dt_obj_now.year, dt_obj_now.month, dt_obj_now.day, 0, 0,
0)))
self.log_info("last update date : " + str(dt_obj_now) +
"\nlast month : " + str(last_month) + "\nLast year : " +
str(last_year) + "\ndate last month : " +
str(dt_last_month) + "\ndays in last month : " +
str(days_in_last_month) + "\ndate today start : " +
str(dt_today_start))
n_events_now, n_events_today, \
n_events_this_month, e_percent_higher_last_month = \
self.get_overall_events_stats(
db_name, dt_now, dt_last_month, dt_today_start)
if n_events_now is None or n_events_today is None or n_events_this_month is\
None or e_percent_higher_last_month is None:
self.log_error(
"Failed to get overall statistics for events so aborting.")
return False
n_mentions_now, n_mentions_today, \
n_mentions_this_month, m_percent_higher_last_month = \
self.get_overall_mentions_stats(
db_name, dt_now, dt_last_month, dt_today_start)
if n_mentions_now is None or \
n_mentions_today is None or \
n_mentions_this_month is None or \
m_percent_higher_last_month is None:
self.log_error(
"Failed to get overall statistics for mentions so aborting.")
return False
n_countries_now, n_countries_today, \
n_countries_this_month, c_percent_higher_last_month = \
self.get_overall_countries_stats(
db_name, dt_now, dt_last_month, dt_today_start)
if n_countries_now is None or \
n_countries_today is None or \
n_countries_this_month is None or \
c_percent_higher_last_month is None:
self.log_error(
"Failed to get overall statistics for countries so aborting.")
return False
update_status = self.dal.update_overall_stats(
db_name, n_events_now, n_events_today, n_events_this_month,
e_percent_higher_last_month, n_mentions_now, n_mentions_today,
n_mentions_this_month, m_percent_higher_last_month,
n_countries_now, n_countries_today, n_countries_this_month,
c_percent_higher_last_month)
return update_status
def get_overall_countries_stats(self, db_name, dt_now, dt_last_month,
dt_today_start):
'''
Method to get overal countries statistics
:param db_name: name of the db
:return: (total_countries_now, total_countries_today,
total_countries_last_month,percentage_higher)
'''
# Get Total Events
n_countries_now = self.dal.get_total_countries(db_name, dt_now)
if n_countries_now == -1: return None, None, None, None
n_countries_till_last_month = self.dal.get_total_countries(
db_name, dt_last_month)
if n_countries_till_last_month == -1: return None, None, None, None
n_countries_this_month = n_countries_now - n_countries_till_last_month
n_countries_today = self.dal.get_total_countries(
db_name, dt_now, start_date=dt_today_start)
if n_countries_today == -1: return None, None, None, None
percent_higher_last_month = 0.0
if n_countries_till_last_month == 0:
percent_higher_last_month = 100.0
else:
percent_higher_last_month = (float(n_countries_this_month) *
100.0) / n_countries_till_last_month
self.log_info("countries till last update : " + str(n_countries_now) +
"\ncountries til last month : " +
str(n_countries_till_last_month) +
"\ncountries this month : " +
str(n_countries_this_month) + "\ncountries today : " +
str(n_countries_today) + "\nIncrease percentage : " +
str(percent_higher_last_month) + "%")
return n_countries_now, n_countries_today, n_countries_this_month, \
percent_higher_last_month
def get_overall_mentions_stats(self, db_name, dt_now, dt_last_month,
dt_today_start):
'''
Method to get overal mentions statistics
:param db_name: name of the db
:return: (total_mentions_now, total_mentions_today,
total_mentions_last_month,percentage_higher)
'''
# Get Total Events
n_mentions_now = self.dal.get_total_mentions(db_name, dt_now)
if n_mentions_now == -1: return None, None, None, None
n_mentions_till_last_month = self.dal.get_total_mentions(
db_name, dt_last_month)
if n_mentions_till_last_month == -1: return None, None, None, None
n_mentions_this_month = n_mentions_now - n_mentions_till_last_month
n_mentions_today = self.dal.get_total_mentions(
db_name, dt_now, start_date=dt_today_start)
if n_mentions_today == -1: return None, None, None, None
percent_higher_last_month = 0.0
if n_mentions_till_last_month == 0:
percent_higher_last_month = 100.0
else:
percent_higher_last_month = (float(n_mentions_this_month) *
100.0) / n_mentions_till_last_month
self.log_info("mentions till last update : " + str(n_mentions_now) +
"\nmentions til last month : " +
str(n_mentions_till_last_month) +
"\nmentions this month : " +
str(n_mentions_this_month) + "\nmentions today : " +
str(n_mentions_today) + "\nIncrease percentage : " +
str(percent_higher_last_month) + "%")
return n_mentions_now, n_mentions_today, n_mentions_this_month, \
percent_higher_last_month
def get_overall_events_stats(self, db_name, dt_now, dt_last_month,
dt_today_start):
'''
Method to get overal events statistics
:param db_name: name of the db
:return: (total_events_now, total_events_last_month, total_events_today,
percentage_higher)
'''
# Get Total Events
n_events_now = self.dal.get_total_events(db_name, dt_now)
if n_events_now == -1: return None, None, None, None
n_events_till_last_month = self.dal.get_total_events(
db_name, dt_last_month)
if n_events_till_last_month == -1: return None, None, None, None
n_events_this_month = n_events_now - n_events_till_last_month
n_events_today = self.dal.get_total_events(db_name,
dt_now,
start_date=dt_today_start)
if n_events_today == -1: return None, None, None, None
percent_higher_last_month = 0.0
if n_events_till_last_month == 0:
percent_higher_last_month = 100.0
else:
percent_higher_last_month = (float(n_events_this_month) *
100.0) / n_events_till_last_month
self.log_info("Events till last update : " + str(n_events_now) +
"\nEvents til last month : " +
str(n_events_till_last_month) +
"\nEvents this month : " + str(n_events_this_month) +
"\nEvents today : " + str(n_events_today) +
"\nIncrease percentage : " +
str(percent_higher_last_month) + "%")
return n_events_now, n_events_today, n_events_this_month, \
percent_higher_last_month
def update_indexes(self, db_name):
'''
Method to update indexes for easier access and sorting
:param db_name: name of the db
:return: update status
'''
update_status = self.dal.update_indexes(db_name)
return update_status
def get_last_month_year(self, dt_now):
'''
Method to get the last month and year
:param dt_now: present datetimeobj
:return: (last_month, last_year)
'''
last_month = dt_now.month - 1
last_year = dt_now.year
if last_month == 0:
last_month = 12
last_year = last_year - 1
return last_month, last_year
def convert_goldstein_to_hex_color(self, g_score):
'''
Method to get a corrsponding color for goldstein's score
:param g_score:
:return:
'''
a1 = 16711680.0
b1 = 16776960.0
a2 = 65280.0
b2 = 16776960.0
color_range_neg = [
"#ff0000", "#ff0000", "#ff3200", "#ff4800", "#ff5d00", "#ff8c00",
"#ff9d00", "#ffbb00", "#ffc300", "#ffd400", "#ffe500"
]
color_range_pos = [
"#ffee00", "#eeff00", "#ddff00", "#b2ff00", "#9dff00", "#76ff00",
"#48ff00", "#3fff00", "#32ff00", "#00ff04", "#00ff04"
]
if g_score == 0.0:
return ("#ffee00")
elif g_score < 0.0:
# val = (((b1-a1)*((g_score+0.2)-(-10.0)))/(0.0-(-10.0))) + a1
# hex_str = str(hex(int(val))).replace("0x","")
# if len(hex_str) < 6:
# while len(hex_str) == 6:
# hex_str = "0" + hex_str
# hex_str = "#" + hex_str
if abs(int(g_score)) > 10:
g_score = -10
hex_str = color_range_neg[int(math.floor(g_score))]
return hex_str
elif g_score > 0.0:
'''
g_score = -1 * g_score
val = (((b2-a2)*((g_score+0.2)-(-10.0)))/(0.0-(-10.0))) + a2
hex_str = str(hex(int(val))).replace("0x","")
if len(hex_str) < 6:
while len(hex_str) == 6:
hex_str = "0" + hex_str
hex_str = "#" + hex_str
'''
if abs(int(g_score)) > 10:
g_score = 10
hex_str = color_range_pos[int(math.ceil(g_score))]
return hex_str
def get_present_date_time(self,
type="gdelt",
d_type="int",
timezone="utc"):
'''
Method to return present date and time in different formats
:param type: type of date (format) ["gdelt" , "default"]
:param d_type: data type of date to return ["int" , "string" , "timestamp"]
:param timezone: which timezone to consider ["utc" , "local"]
:return: datetime in the selected format and data type
'''
dt_now = None
if timezone == "utc":
dt_now = datetime.utcnow()
elif timezone == "local":
dt_now = datetime.now()
else:
dt_now = datetime.now()
if type == "gdelt":
if d_type == "string":
return dt_now.strftime("%Y%m%d%H%M%S")
elif d_type == "int":
return int(dt_now.strftime("%Y%m%d%H%M%S"))
elif d_type == "tuple":
return dt_now
else:
return dt_now.strftime("%Y%m%d%H%M%S")
elif type == "default":
if d_type == "string":
return dt_now.strftime("%Y-%m-%d %H:%M:%S")
elif d_type == "timestamp":
return time.mktime(
time.strptime(dt_now.strftime("%Y%m%d%H%M%S"),
"%Y%m%d%H%M%S"))
else:
return dt_now.strftime("%Y-%m-%d %H:%M:%S")
else:
return dt_now.strftime("%Y-%m-%d %H:%M:%S")
def gdelt_date_to_datetime(self, gdelt_date):
'''
Method to convert date in GDELT format to python datetime object
:param gdelt_date: date in gdelt format in integer
:return: corresponding datetime
'''
return datetime.strptime(str(gdelt_date), "%Y%m%d%H%M%S")
def datetime_to_gdelt_date(self, datetime):
'''
convert python datetime object to GDELT date
:param datetime: datetime object
:return: GDELT date in int
'''
return int(datetime.strftime("%Y%m%d%H%M%S"))
def delete_files(self, files):
'''
Method to remove files from wrodking directory
:param files: list of file paths
:return:
'''
for file in files:
try:
os.remove(file)
except Exception as e:
print("could not remove file " + file + "\nException: " +
e.message)
def download_file(self, working_dir, url):
'''
Method for downloading files from a URL
:param working_dir: directory to download the file to
:param url:
:return: download_status, downloaded_file_path
'''
# Open the url
self.log_info("Downloading file from url " + url + " to " +
working_dir)
try:
f = urlopen(url)
print("downloading " + url)
filename = url.split("/")[-1]
download_path = os.path.join(working_dir, filename)
# Open our local file for writing
with open(download_path, "wb") as local_file:
local_file.write(f.read())
return True, download_path
# handle errors
except HTTPError, e:
self.log_error(
"Exception occurred while Downloading file from URL: " +
str(url) + ".\n" + "Exception stacktrace: \n" +
traceback.format_exc())
# self.log_error("HTTP Error:"+ str(url))
return False, None
except URLError, e:
self.log_error(
"Exception occurred while downloading file from url: " +
str(url) + "\n" + "Exception stacktrace: \n" +
traceback.format_exc())
return False, None
def api_top_list():
dal = Dal()
data_set = dal.top_list()
if len(data_set) is 0:
return "No posts in database"
return json.dumps(data_set), 200, {'ContentType': 'application/json'}
def update_post(post_id, post_text):
dal = Dal()
if not dal.update(post_id, post_text):
return http_error('error while inserting record to storage', 500)
return http_error('success', 200)
def create_post(post_text):
dal = Dal()
if not dal.create(post_text):
return http_error('error while inserting record to storage', 500)
else:
return http_error('success', 200)
from flask import Flask, request, make_response
from flask_cors import CORS
from bson.json_util import dumps
import json
import config
from dal import Dal
app = Flask(__name__)
cors = CORS(app)
db = Dal(config.DB['HOST'], config.DB['PORT'])
@app.route('/login', methods=['POST'])
def login():
credentials = json.loads(request.data)
return_object = {'email': credentials['email'], 'name': 'Tomer', 'token': 'kaki'}
return make_response(json.dumps(return_object), 200)
@app.route('/carts/<user>')
def getAllCarts(user):
carts = db.getAllCarts(user)
return make_response(dumps(carts), 200)
if __name__ == '__main__':
app.run(host=config.HOST, port=config.PORT)