# input parser = argparse.ArgumentParser() parser.add_argument("-config", dest="CONFIG_FILE", default="config-sample.json", help="Config file") a = parser.parse_args() config = io.readJSON(a.CONFIG_FILE) configContent = config["content"] OUTPUT_DIR = "apps/{appname}/".format(appname=config["name"]) OUTPUT_SET_DIR_REL = "data/sets/" OUTPUT_SET_DIR = OUTPUT_DIR + OUTPUT_SET_DIR_REL CONFIG_FILE = OUTPUT_DIR + "js/config/config.sets.js" # Make sure output dirs exist io.makeDirectories([OUTPUT_SET_DIR, CONFIG_FILE]) sets, items = tu.getItems(config) # Remove existing data io.removeFiles(OUTPUT_SET_DIR + "*.json") jsonsets = {} for keyName, options in configContent.items(): if "query" not in options: continue setItems = lu.filterByQueryString(items, options["query"]) if len(setItems) > 0: print("%s results found for '%s'" % (len(setItems), options["query"])) else:
dest="OUTPUT_FILE", default="reports/MexicoAndCentralAmerica.txt", help="Output text file") parser.add_argument('-detail', dest="DETAILED_OUTPUT_FILE", default="", help="Output each field as an individual csv file") a = parser.parse_args() FIELDS_HAS = a.FIELDS_HAS.strip().split(",") FIELDS_COUNTS = a.FIELDS_COUNTS.strip().split(",") FIELDS_LISTS = a.FIELDS_LISTS.strip().split(",") FIELDS_MERGE = io.parseQueryString(a.FIELDS_MERGE.strip()) # Make sure output dirs exist io.makeDirectories([a.OUTPUT_FILE]) if len(a.DETAILED_OUTPUT_FILE) > 0: io.makeDirectories([a.DETAILED_OUTPUT_FILE]) items = [] fieldNames = [] if "*" in a.INPUT_FILE: files = glob.glob(a.INPUT_FILE) for fn in files: fFieldNames, fItems = io.readCsv(fn) fieldNames += fFieldNames items += fItems fieldNames = lu.unique(fieldNames)
dest="INPUT_FILE", default="data/MexicoAndCentralAmerica.csv", help="File generated by html_to_csv.py") parser.add_argument('-field', dest="FIELD", default="Locale", help="Field to output") parser.add_argument( '-out', dest="OUTPUT_FILE", default="data/processed/MexicoAndCentralAmerica_locales.csv", help="Output csv file") a = parser.parse_args() # Make sure output dirs exist io.makeDirectories([a.OUTPUT_FILE]) fieldNames, items = io.readCsv(a.INPUT_FILE) itemCount = len(items) values = [item[a.FIELD] for item in items] counter = collections.Counter(values) counts = counter.most_common() rows = [] for value, count in counts: if len(str(value).strip()) < 1: continue row = {} row[a.FIELD] = value row["Count"] = count
maxCellWidth = 512 containsAlpha = config["imageHasAlpha"] if "imageHasAlpha" in config else False defaultColor = config["defaultColor"] if "defaultColor" in config else "#3C3C3C" filenameKey = config[ "filenameColumn"] if "filenameColumn" in config else "filename" imageDir = config["imageDirectory"] noImageValue = config["noImageValue"] if "noImageValue" in config else None OUTPUT_DIR = "apps/{appname}/".format(appname=config["name"]) OUTPUT_TEXTURES_DIR_REL = "img/textures/" OUTPUT_TEXTURES_DIR = OUTPUT_DIR + OUTPUT_TEXTURES_DIR_REL CONFIG_FILE = OUTPUT_DIR + "js/config/config.textures.js" if not a.PROBE: # Make sure output dirs exist io.makeDirectories([OUTPUT_TEXTURES_DIR, CONFIG_FILE, a.CACHE_DIR]) # Remove existing images io.removeFiles(OUTPUT_TEXTURES_DIR + "*.jpg") items, categories = tu.getItems(config) # Make texture for each set # sets = list(configSets.items()) sets = [] # just produce the default set for now sets = [("default", {"query": ""})] + sets # add default set jsonsets = {} for keyName, options in sets: setItems = lu.filterByQueryString(items, options["query"]) if len(setItems) > 0: print("%s results found for '%s'" % (len(setItems), options["query"]))
"Cache-Control": "max-age=0", "Connection": "keep-alive", # "Content-Length": "123", "Content-Type": "application/x-www-form-urlencoded", "Host": "anthro.amnh.org", "Origin": "https://anthro.amnh.org", "Referer": "https://anthro.amnh.org/anthropology/databases/common/query_categories.cfm?", "Sec-Fetch-Mode": "nested-navigate", "Sec-Fetch-Site": "same-origin", "Sec-Fetch-User": "******", "Upgrade-Insecure-Requests": "1", "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36" } # Make sure output dirs exist io.makeDirectories([a.HTML_DIR]) if a.OVERWRITE: io.removeFiles(a.HTML_DIR % '*') page = 1 zeroPadding = 4 perPage = query["rec_per_page"] totalPages = None totalRecords = None currentRecord = None prevPageQuery = None while True: postData = query.copy() if prevPageQuery is None else prevPageQuery filename = a.HTML_DIR % str(page).zfill(zeroPadding)
dest="MAX_SUBJECTS", default=64, type=int, help="Max number of subjects (includes 'other')") parser.add_argument('-out', dest="OUTPUT_FILE", default="data/photographic_images.json", help="File for output") a = parser.parse_args() YEAR_RANGE = [1600, 2020] gridW, gridH = tuple([int(t) for t in a.GRID_SIZE.split("x")]) # Make sure output dirs exist io.makeDirectories(a.OUTPUT_FILE) # retrieve data fieldNames, data = io.readCsv(a.INPUT_FILE) dataCount = len(data) _, subjectData = io.readCsv(a.SUBJECTS_FILE) grid = np.loadtxt(a.GRID_FILE, delimiter=",") imageFiles = glob.glob(a.IMAGE_FILES) imageFiles = sorted(imageFiles) fileCount = len(imageFiles) print("Loaded %s files" % fileCount) # process subject data subjectData = groupList(subjectData, "subject", sort=True) subjectCount = len(subjectData) mainSubjects = subjectData[:a.
import lib.list_utils as lu import lib.math_utils as mu # input parser = argparse.ArgumentParser() parser.add_argument('-in', dest="INPUT_FILE", default="data/MexicoAndCentralAmerica.csv", help="File generated by html_to_csv.py") parser.add_argument('-out', dest="OUTPUT_DIR", default="images/MexicoAndCentralAmerica/", help="Output dir") parser.add_argument('-overwrite', dest="OVERWRITE", action="store_true", help="Overwrite existing data?") a = parser.parse_args() # Make sure output dirs exist io.makeDirectories([a.OUTPUT_DIR]) fieldNames, items = io.readCsv(a.INPUT_FILE) itemCount = len(items) for i, item in enumerate(items): imgUrl = item['Thumb URL'] if len(imgUrl) > 0: io.downloadBinaryFile(imgUrl, a.OUTPUT_DIR, overwrite=a.OVERWRITE) print("%s%%" % round((i + 1) / len(items) * 100.0, 2))
a = parser.parse_args() config = tu.loadConfig(a.CONFIG_FILE) itemFields = config["itemFields"] if "itemFields" in config else None if itemFields is None: print(f'No item metadata fields are set in config. Skipping...') sys.exit() OUTPUT_DIR = f'apps/{config["name"]}/' OUTPUT_FILE_REL = "data/metadata/" OUTPUT_FILE = OUTPUT_DIR + OUTPUT_FILE_REL CONFIG_FILE = OUTPUT_DIR + "js/config/config.metadata.js" # Make sure output dirs exist io.makeDirectories([OUTPUT_FILE, CONFIG_FILE]) io.removeFiles(OUTPUT_FILE + "*.json") items, categories = tu.getItems(config) itemCount = len(items) # Add item slides from stories to item metadata storyItems = {} if "stories" in config: stories = config["stories"] for key, story in stories.items(): for slide in story["slides"]: if "itemId" in slide: slideMeta = slide.copy() slideMeta.pop("itemId", None) storyItems[str(slide["itemId"])] = slideMeta
"countries": { "fontSize": config["labels"]["countryFontSize"], "thickness": config["labels"]["countryFontThickness"], "defaultView": "geographyBars", "layout": "bars" } } PRECISION = 5 OUTPUT_DIR = "apps/{appname}/".format(appname=config["name"]) OUTPUT_LABEL_DIR_REL = "data/labels/" OUTPUT_LABEL_DIR = OUTPUT_DIR + OUTPUT_LABEL_DIR_REL CONFIG_FILE = OUTPUT_DIR + "js/config/config.labels.js" # Make sure output dirs exist io.makeDirectories([OUTPUT_LABEL_DIR, CONFIG_FILE]) # Remove existing data io.removeFiles(OUTPUT_LABEL_DIR + "*.json") items, categories = tu.getItems(config) itemCount = len(items) def getYearLabels(userOptions={}): global items cfg = {} options = {"y": 0.667} options.update(userOptions) yearCol = "year" if yearCol not in items[0]:
parser.add_argument('-img', dest="IMAGE_FILE", default="images/{Region}/{Filename}", help="Input image file pattern") parser.add_argument('-cache', dest="CACHE_FILE", default="tmp/imageCache_16.p.gz", help="Input image file pattern") parser.add_argument('-out', dest="OUTPUT_FILE", default="data/viz/timeline%s.jpg", help="Output image filename") parser.add_argument('-count', dest="IMAGE_COUNT", default=4, type=int, help="Number of images to produce") parser.add_argument('-height', dest="IMAGE_HEIGHT", default=2160, type=int, help="Target height") parser.add_argument('-ywidth', dest="YEAR_WIDTH", default=100, type=int, help="Target height") parser.add_argument('-gwidth', dest="ITEM_GROUP_WIDTH", default=4.0, type=float, help="Max width of the an item group as a percentage of min(year width, region height)") parser.add_argument('-plot', dest="PLOT", action="store_true", help="Plot the data?") parser.add_argument('-probe', dest="PROBE", action="store_true", help="Just output details?") a = parser.parse_args() COLORS = ["#612323", "#204f1c", "#4d1e59", "#112e6b", "#4b5713", "#571330"] colorCount = len(COLORS) # Make sure output dirs exist io.makeDirectories([a.OUTPUT_FILE, a.CACHE_FILE]) font = ImageFont.truetype(font="fonts/Open_Sans/OpenSans-Regular.ttf", size=a.FONT_SIZE) print("Reading data...") fieldNames, items = io.readCsv(a.INPUT_FILE) yLabels = lu.unique([item[a.Y_AXIS] for item in items]) if a.Y_AXIS == "Region": items = [item for item in items if item["Region"] != "Europe"] itemsByRegion = lu.groupList(items, "Region") for i, region in enumerate(itemsByRegion): itemsByRegion[i]["lat"] = np.mean([item["Latitude"] for item in region["items"] if -90 <= item["Latitude"] <= 90]) itemsByRegion = sorted(itemsByRegion, key=lambda region: -region["lat"]) yLabels = [region["Region"] for region in itemsByRegion] else: yLabels = sorted(yLabels)