def create_list_datetimes(self, start_date, end_date, nhours): """ Create a list of timesteps between start_date and end_date with a time interval of nhours. Add timesteps to database """ from datetime import datetime from datetime import timedelta if not (isinstance(start_date, datetime) and isinstance(end_date, datetime)): message = ( "input start_date and end_date of function ", "create_list_datetimes should be datetime.datetime objects", ) logger.error(message) raise IOError(message) if not (isinstance(nhours, int)): message = ("nhours input argement of create_list_datetimes should be ", "an integer number of hours") logger.error(message) raise IOError(message) # check if the frequency between boundary conditions is not larger than the # period between start and end date of the simulation if (nhours * 3600) > (end_date - start_date).total_seconds(): message = ( "Interval of boundary conditions larger than timedelta ", "between start_date and end_date in function ", "database.create_list_datetimes", ) logger.error(message) raise IOError(message) timesteps = utils.datetime_range(start_date, end_date, {"hours": nhours}) for idx, timestep in enumerate(timesteps): self._add_timestep_to_db(timestep)
def create_list_datetimes(self, start_date, end_date, nhours): ''' Create a list of timesteps between start_date and end_date with a time interval of nhours. Add timesteps to database ''' from datetime import datetime from datetime import timedelta if not(isinstance(start_date, datetime) and isinstance(end_date, datetime)): message = ('input start_date and end_date of function ', 'create_list_datetimes should be datetime.datetime objects') logger.error(message) raise IOError(message) if not(isinstance(nhours, int)): message = ('nhours input argement of create_list_datetimes should be ', 'an integer number of hours') logger.error(message) raise IOError(message) # check if the frequency between boundary conditions is not larger than the # period between start and end date of the simulation if ((nhours*3600) > (end_date - start_date).total_seconds()): message = ('Interval of boundary conditions larger than timedelta ', 'between start_date and end_date in function ', 'database.create_list_datetimes') logger.error(message) raise IOError(message) timesteps = utils.datetime_range(start_date, end_date, {'hours':nhours}) for idx,timestep in enumerate(timesteps): self._add_timestep_to_db(timestep)
def main(): path_file_base = os.path.join('bases', 'idka_base.csv') # faz o download do csv no site da anbima url = 'http://www.anbima.com.br/informacoes/idka/IDkA-down.asp' name_download_folder = 'idka' path_download = utils.prepare_download_folder(name_download_folder) # verifica a última data disponível na base today = datetime.now().date() cal = utils.get_calendar() ultima_data_base = cal.offset(today, -6) dates_range = list(utils.datetime_range(start=ultima_data_base, end=today)) for dt_referencia in reversed(dates_range): path_file = os.path.join( path_download, dt_referencia.strftime('%Y%m%d') + '_idka.csv') download_file(url, dt_referencia, path_file) utils.remove_zero_files(name_download_folder) import_files(name_download_folder, path_file_base, ultima_data_base) # organizar o arquivo base por dt_referencia utils.generate_csv_base(path_file_base) print("Arquivos baixados com sucesso e importados para a base de dados")
def main(): # apaga arquivos antigos remove_old_files() # verifica a última data disponível na base name_file_base = 'debentures_base.csv' path_file_base = os.path.join('bases', name_file_base) # verifica a última data disponível na base today = datetime.now().date() cal = utils.get_calendar() ultima_data_base = cal.offset(today, -6) dates_range = list(utils.datetime_range(start=ultima_data_base, end=today)) # faz o download do csv no site da anbima url = 'https://www.anbima.com.br/informacoes/merc-sec-debentures/arqs/db' for dt_referencia in reversed(dates_range): path_download = os.path.join('downloads') if not os.path.exists(path_download): os.makedirs(path_download) path_download = os.path.join('downloads', 'debentures') if not os.path.exists(path_download): os.makedirs(path_download) file_path = os.path.join(path_download, dt_referencia.strftime('%y%m%d') + '.txt') # faz o download do arquivo caso ele ainda não tiver sido baixado if not os.path.exists(file_path): download_file(url, dt_referencia, file_path) print("Arquivos baixados com sucesso")
def main(): parser = OptionParser() parser.add_option('-o', dest='filename', default='retweet_timeseries.csv', help='output filename') parser.add_option('--clusters', dest='clusters', default='data/clusters.json', help='clusters filename') parser.add_option('--start', dest='start', default='20110301', help='start date (default=20110301)') parser.add_option('--stop', dest='stop', default='20130401', help='stop date (default=20130401)') parser.add_option('--influencers', dest='n', type='int', default=None, help='maximum number of influencers (default=inf)') parser.add_option('--mincount', dest='mincount', type='int', default=1, help='minimum number of retweet (default=1)') options, args = parser.parse_args() start_date = ymd_to_datetime(options.start) stop_date = ymd_to_datetime(options.stop) cluster_map = load_clusters(options.clusters) clusters = list(set(cluster_map.values())) days = [ d.strftime('%Y%m%d%H') for d in datetime_range(start_date, stop_date, timedelta(hours=1)) ] count = {} for c in clusters: count[c] = {} for d in days: count[c][d] = 0 tweets = load_tweet(*args) tweets = filter_by_datetime(tweets, start_date, stop_date) for ruid, tss in group_by_influencer(tweets, options.n, options.mincount): if ruid in cluster_map: cluster = cluster_map[ruid] for day, ts in group_by_hour(tss): count[cluster][day] += len(list(ts)) writer = csv.writer(open(options.filename, 'w')) writer.writerow(['date'] + clusters) for d in days: writer.writerow([d] + [count[c][d] for c in clusters])
def fill_gaps(time_begin, time_end, result): # 如果这一分钟没有值,用上一分钟的值 step = datetime.timedelta(seconds=60) first_minute = truncate_minute(time_begin) first_exist = min(result) last = result[first_exist] if first_exist > first_minute: logging.warning("data missing from %s to %s", first_minute, first_exist) for t in datetime_range(first_minute, time_end + step, step): last = result.setdefault(t, last)
def main(): # apaga arquivos antigos remove_old_files() # verifica a última data disponível na base name_file_base = 'ima_quadro_resumo_base.csv' path_file_base = os.path.join('bases', name_file_base) # ultima data base dispon[ivel ultima_data_base = get_ultima_data_disponivel_base(path_file_base) print('Última data base disponível:', ultima_data_base) if (ultima_data_base is None): ultima_data_base = datetime.date(2010, 11, 17) carteiras = [ 'irf-m', 'irf-m 1', 'irf-m 1+', 'ima-b', 'ima-b 5', 'ima-b 5+', 'ima-c', 'ima-s', 'ima-geral', 'ima-geral ex-c' ] # faz o download do csv no site da anbima url = 'http://www.anbima.com.br/informacoes/ima/ima-carteira-down.asp' # verifica a última data disponível na base today = datetime.now().date() cal = utils.get_calendar() ultima_data_base = cal.offset(today, -5) dates_range = list(utils.datetime_range(start=ultima_data_base, end=today)) for dt_referencia in reversed(dates_range): for carteira in carteiras: path_download = os.path.join('downloads', carteira) if not os.path.exists(path_download): os.makedirs(path_download) file_name = os.path.join( path_download, dt_referencia.strftime('%Y%m%d') + '_' + carteira + '.csv') if utils.check_download(dt_referencia, file_name) is False: break # faz o download do arquivo caso ele ainda não tiver sido baixado if not os.path.exists(file_name): download_file_carteira(url, dt_referencia, file_name, carteira) print("Arquivos baixados com sucesso e importados para a base de dados")
def main(): # apaga arquivos antigos remove_old_files() # verifica a última data disponível na base name_file_base = 'ima_quadro_resumo_base.csv' path_file_base = os.path.join('bases', name_file_base) # ultima data base dispon[ivel ultima_data_base = get_ultima_data_disponivel_base(path_file_base) print('Última data base disponível:', ultima_data_base) if (ultima_data_base is None): ultima_data_base = datetime.date(2010, 11, 17) # faz o download do csv no site da anbima # lft url = 'http://www.anbima.com.br/informacoes/merc-sec/arqs/ms' # verifica a última data disponível na base today = datetime.now().date() cal = utils.get_calendar() ultima_data_base = cal.offset(today, -5) dates_range = list(utils.datetime_range(start=ultima_data_base, end=today)) path_download = os.path.join('downloads', 'titulos-publicos-merc-secundario') if not os.path.exists(path_download): os.makedirs(path_download) for dt_referencia in reversed(dates_range): file_path = os.path.join( path_download, dt_referencia.strftime('%Y%m%d') + '_ms_titulos-publicos.txt') # verifica se o arquivo deve ser baixado if not utils.check_download(dt_referencia, file_path): continue print(file_path) # faz o download do arquivo caso ele ainda não tiver sido baixado if not os.path.exists(file_path): download_file(url, dt_referencia, file_path) print("Arquivos baixados com sucesso")
def gamelogs(): gamelogs_query = {} name = request.args.get('name') if name: gamelogs_query.update({'Player': name}) start = request.args.get('start') end = request.args.get('end') if start: gamelogs_query.update(datetime_range(start, end)) active = request.args.get('active') if not active: gamelogs_list = query.query_games(gamelogs_query) else: gamelogs_list = query.query_games(gamelogs_query, active=True) if gamelogs_list: for gamelog_dict in gamelogs_list: gamelog_dict.pop("_id", None) return Response(json.dumps(gamelogs_list, default=json_util.default), mimetype='application/json')
print('Fitting') stepwise_model.fit(dataset) print('Done!') return stepwise_model if __name__ == "__main__": start_time_stamp = "2020-04-06T00:00:00" end_time_stamp = "2020-04-21T00:00:00" start = datetime.strptime(start_time_stamp, STR_FMT) end = datetime.strptime(end_time_stamp, STR_FMT) dts = [ dt.strftime(STR_FMT) for dt in datetime_range(start, end, timedelta(hours=1)) ] df = read_data() list_of_time_series = [] # df.to_csv('test.csv') for i in set(df["CAMERA_ID"]): list_of_time_series.append(df[df["CAMERA_ID"] == i].drop( columns=["CAMERA_ID", "LAT", "LONG"])) # list_of_models = [] # predictions = [] # for time_series in tqdm(list_of_time_series): # model = train_ARIMA(time_series) # list_of_models.append(model) # predictions.append(model.predict(24)) # time_stamps = sorted(list(set(df.index)))
from keras_yolo.yolo import YOLO from utils import datetime_range, get_data, save_image_from_url STR_FMT = '%Y-%m-%dT%H:%M:%S' if __name__ == "__main__": start_time_stamp = "2020-03-18T20:00:00" end_time_stamp = "2020-03-21T00:00:00" refresh_rate = 60 * 60 start = datetime.strptime(start_time_stamp, STR_FMT) end = datetime.strptime(end_time_stamp, STR_FMT) dts = [ dt.strftime(STR_FMT) for dt in datetime_range(start, end, timedelta(seconds=refresh_rate)) ] od = YOLO() # main loop with open( os.path.join('results', start_time_stamp + '_to_' + end_time_stamp + '.json'), 'w') as json_file: result_dict = {} for time_stamp in tqdm(dts): with open(os.path.join('results', time_stamp + '.csv'), 'w') as f: f.write('CAMERA_ID,LAT,LONG,NUM_OF_VEH\n') result_dict[time_stamp] = {} # print('Time stamp: ',time_stamp) res = get_data(time_stamp)