def scanfolder(data, filepath): if not data: data = {} filelist = data.get('files', []) scannedpaths = data.get('scanned', {}) if not filepath in scannedpaths: scan = { "filepath": filepath, "scanned": datetime.datetime.now(), } found = 0 for f in scanfiles(filepath): filelist.append(f) found += 1 logger.info('Found {} / {} files'.format(found, len(filelist))) scan['found'] = found data['files'] = filelist scannedpaths[filepath] = scan data['scanned'] = scannedpaths return data
def main(): logger = logging.getLogger(__name__) PATHS = {'LOCAL': os.getcwd(), 'HOME': os.path.expanduser("~")} config = load_config('ktapi.json') try: logging.config.dictConfig(config.get('logger', {})) except: logging.basicConfig(level=logging.INFO) config["paths"] = PATHS folderpath = get_folderpath(config, '_config', PATHS) logging.info("Scanning folder: {}".format(folderpath)) for f in scanfiles(folder=folderpath, filter=".pickle$"): filepath = os.path.join(f.get('folder', ''), f.get('file')) data = get_pickle_data(pickleName=filepath) for k, v in data.items(): if v: logger.info("\t{} => {} : {}".format(k, type(v), len(v))) else: logger.info("\t{}".format(k)) content = data.get('content') if content: filename, _ = os.path.splitext(f.get('file')) filename = '{}.json'.format(filename) save_json(file_path=filename, data=content) logger.info("Export: {} => {}".format('content', filename))
logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) homepath = os.path.expanduser("~") datapaths = ["OneDrive - Great Canadian Railtour Co", "Jupyter_NB"] savepath = os.path.join(homepath, *datapaths) filter_str = "[a-zA-Z_]*.json" pickle_file = "kaptio_dumps.pickle" data = get_pickle_data(pickle_file) data['files'] = [] data['names'] = {} for f in scanfiles(savepath): data['files'].append(f) if not f['file'] in data['names']: data['names'][f['file']] = [] data['names'][f['file']].append(f) logger.info("found {} files.".format(len(data['names']))) save_pickle_data(data, pickle_file) #savepath = r"c:\Users\dgloyncox\git\dreffed\Kaptio_API" logger.info("scanning folder {}".format(savepath)) for dirpath, _, filenames in os.walk(savepath): logger.info(dirpath, len(filenames))
season_end = data['season']['end'] kt_packages = data['packages'] kt_pricelist = data['pricelist'] rev_occ = {} for key, value in occupancy.items(): rev_occ[value] = key # get teh file list datapath = localpath #os.path.join(savepath, 'data') fn_pricelist = [] fn_reprocess = [] filecount = 0 errorcount = 0 logger.info("scanning... {}".format(datapath)) for f in scanfiles(datapath, r'price_([a-zA-Z0-9]+)_([\d]+)\.json'): filepath = os.path.join(f['folder'], f['file']) try: filecount += 1 with open(filepath, mode='r') as fp: try: pl_data = json.load(fp) fn_pricelist.append(pl_data) continue except Exception as ex: pass try: pl_data = json.loads("[{}]".format(fp.read().replace("}{", "},{"))) fn_pricelist.append(pl_data) except Exception as ex:
import os from time import time from datetime import datetime import logging logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) update_lookups = False homepath = os.path.expanduser("~") datapaths = ["OneDrive - Great Canadian Railtour Co", "Jupyter_NB"] savepath = os.path.join(homepath, *datapaths) timestamp = datetime.now().strftime("%Y%m%d%H%M%S") pickle_files = scanfiles('.', r".*\.pickle") logger.info("Pickles:") for f in pickle_files: logger.info("\t{} => {} [{}]".format(f['file'], f['folder'], json.dumps(f, indent=2))) pickle_file = "kaptio_allsell.pickle" data = get_pickle_data(pickle_file) for key, value in data.items(): logger.info(key) if update_lookups: tax_profiles = { "Zero Rated": "a8H4F0000003tsfUAA",
homepath = os.path.expanduser("~") datapaths = ["OneDrive - Great Canadian Railtour Co", "Jupyter_NB"] savepath = os.path.join(homepath, *datapaths) localpaths = ["data"] localpath = os.path.join(homepath, *localpaths) if not os.path.exists(localpath): # creeate the folder... logger.info("Creating archive directory: {}".format(localpath)) os.makedirs(localpath) datestamp = datetime.now() datapath = os.path.join(savepath, "data") filecount = 0 for f in scanfiles(datapath, r".*\.json"): filecount += 1 filedate = datetime.strptime(f['modified'], "%Y-%m-%d %H:%M:%S") filepath = os.path.join(f.get('folder'), f.get('file')) if (datestamp - filedate).total_seconds() > (8 * 60 * 60): os.remove(filepath) continue destfolder = os.path.join(localpath, filedate.strftime("%Y-%m-%d")) if not os.path.exists(destfolder): os.makedirs(destfolder) destfile = os.path.join(destfolder, f.get('file')) # copy the file over... try: shutil.move(filepath, destfile) except FileExistsError:
def main(): PATHS = {'LOCAL': os.getcwd(), 'HOME': os.path.expanduser("~")} config = load_config('ktapi.json') try: logging.config.dictConfig(config.get('logger', {})) except: logging.basicConfig(level=logging.INFO) config["paths"] = PATHS scan_local = config.get("flags", {}).get("switches", {}).get("scan_local", False) scan_remote = config.get("flags", {}).get("switches", {}).get("scan_remote", False) get_remote_content = config.get("flags", {}).get("switches", {}).get("import_remote", False) check_updates = config.get("flags", {}).get("switches", {}).get("check_updates", False) savepath = get_folderpath(config, '_remote', PATHS) logger.info('Savepath: {}'.format(savepath)) timestamp = datetime.now().strftime("%Y%m%d%H%M%S") logger.info("Timestamp: {}".format(timestamp)) logger.info( "Runnins:\n\tscan local\t{}\n\tscan remote\t{}\n\tget remote\t{}\n\tcheck updates\t{}" .format(scan_local, scan_remote, get_remote_content, check_updates)) if scan_local: logger.info("Local Pickles:") for f in scanfiles('.', r".*\.pickle"): logger.info("\t{} => {}".format(f['file'], f['folder'])) if scan_remote: logger.info("Remote Pickles:") for f in scanfiles(os.path.join(savepath, 'config'), r".*\.pickle"): logger.info("\t{} => {}".format(f['file'], f['folder'])) currencies = config.get("presets", {}).get("currencies", ["CAD"]) for currency in currencies: config['presets']['currency'] = currency config_type = config.get("configurations", {}).get("run", {}).get("pickle") pickle_file = get_configuration_path(config, config_type, PATHS) name, ext = os.path.splitext(pickle_file) pickle_file = "{}_{}{}".format(name, currency, ext) logger.info("Loading pickle file {}".format(pickle_file)) if not 'presets' in config: config['presets'] config['presets']['pickle'] = pickle_file data = get_pickle_data(pickle_file) if get_remote_content: config_type = config.get("configurations", {}).get("run", {}).get("remote_pickle") if not config_type: logger.error( "\tUnable to local remote pickle details {configurations->run->remote_pickle}" ) break remote_pickle = get_configuration_path(config, config_type, config.get('paths', [])) logger.info("\tloading remote {}".format(remote_pickle)) data_src = get_pickle_data(remote_pickle) if 'content' in data_src: logger.info("Fetching remote cached content") kt_content = data_src.get('content') if kt_content: data['content'] = kt_content logger.info("Retrieved remote cached content") save_pickle_data(data, pickle_file) display_data(pickle_file=pickle_file, data=data, name="Local") if check_updates: update_prices(data=data, data_src=data_src)