def main(): inputArgs = sys.argv args = inputArgs[1:] yearsTempList, magnitudeOver, overwrite = Input.getValues(args) years = Database.queryInput(yearsTempList, magnitudeOver, overwrite) StoreData.createFolder() print "Requesting earthquakes data with magnitude over {}, for years: {}".format(magnitudeOver, years) for year in years: print "Processing year: ", year print "Data acquisition starts" firstDate = date(year, 1, 1) lastDate = date(year, 12, 31) for d in dateRange(firstDate, lastDate): start = d.strftime("%Y-%m-%d") + "T00:00:00.000Z" end = (d + timedelta(days=1)).strftime("%Y-%m-%d") + "T00:00:00.000Z" try: eq_list_raw = Acquisition.request(start, end, magnitudeOver) eq_list_temp = Preprocessing.cleanHeaders(eq_list_raw) eq_list = Preprocessing.splitDateTime(eq_list_temp) StoreData.toFile(eq_list, year, d,magnitudeOver) except Exception as error: print "Error while processing a request:", error print "Data acquisition ended" path = HDFS.getPath() HDFS.put('../data/earthquakes{}mag{}.csv'.format(year, magnitudeOver), path)
def main(): Log.info('-----------------------') Log.info('Download process starts') Log.info('-----------------------') inputArgs = sys.argv args = inputArgs[1:] StoreData.createFolder() yearsTempList, magnitudeOver, download_again = Input.getValues(args) path = HDFS.getPath() Log.info("Earthquakes acquisition starts..") years = Database.QueryInput(yearsTempList, magnitudeOver, download_again) Log.info( "Requesting earthquakes data with magnitude over {}, for years: {}". format(magnitudeOver, years)) for year in years: Log.info("Processing year: {}".format(year)) Log.info("Earthquakes acquisition starts.") firstDate = date(year, 1, 1) lastDate = date(year, 12, 31) for d in dateRange(firstDate, lastDate): start = d.strftime("%Y-%m-%d") + "T00:00:00.000Z" end = (d + timedelta(days=1)).strftime("%Y-%m-%d") + "T00:00:00.000Z" try: eq_list_raw = Acquisition.Request(start, end, magnitudeOver) eq_list_no_headers = Preprocessing.cleanHeaders(eq_list_raw) eq_list_split_date_time = Preprocessing.splitDateTime( eq_list_no_headers) eq_list = Preprocessing.checkCountry(eq_list_split_date_time) StoreData.toFile(eq_list, year, d, magnitudeOver) except Exception as error: Log.error("Error while processing a Request:") Log.error(error) Log.info("Earthquakes acquisition for year {} finished".format(year)) HDFS.put( '../../data/earthquakes-history/earthquakes{}mag{}.csv'.format( year, magnitudeOver), path) Log.info('---------------------') Log.info('Download process ends') Log.info('---------------------')
def main(): StoreData.createFolder() try: interval = int(sys.argv[1]) print "Interval parameter passed: ", interval except: interval = 10 print "No interval parameter passed, interval default value ", interval end = datetime.utcnow() - timedelta(minutes=10) start = end - timedelta(minutes=interval) while True: print "Data acquisition starts" print "Requesting earthquakes data" print "from ", start print "to ", end eq_list_raw = Acquisition.request(start, end) eq_list_temp = Preprocessing.cleanHeaders(eq_list_raw) eq_list = Preprocessing.splitDateTime(eq_list_temp) StoreData.toFile(eq_list) print "Data acquisition ended" print "Process starts again in {} minutes".format(interval) time.sleep(interval * 60) end = datetime.utcnow() - timedelta(minutes=10) start = end - timedelta(minutes=interval)