def export_dataframe_to_csv(name: str, dataframe): filename = os.path.join(CSV_FOLDER, f'{name}.csv') old_hash = sha1sum(filename) dataframe.replace({ 0: None }).astype('Int64').to_csv(filename, line_terminator='\r\n') write_timestamp_file(filename=filename, old_hash=old_hash)
def login(self, username, password): cookies = self.load_cookies() if self.test_cookies(cookies): return password = utils.sha1sum(password) data = {'username': username, 'sha1': password, 'op': 'login'} r = self.post(data) if 'You need to login before proceed!' not in r.text: self.cookies = r.cookies self.dump_cookies()
export_dataframe_to_csv(name='municipality-confirmed', dataframe=df.cumsum()) df = df.rolling(min_periods=1, window=14).sum().replace({0: None}).astype('Int64') \ .drop('region.n.neznano', axis='columns') export_dataframe_to_csv(name='municipality-active', dataframe=df) with open(file=os.path.join(CSV_FOLDER, 'dict-municipality.csv'), encoding='utf-8') as f: for row in csv.DictReader(f): municipalities[row['name'].lower()] = row if row['name_alt']: municipalities[row['name_alt'].lower()] = row # --- municipality-deceased.csv --- # Copy paste latest row for every missing date municipality_deceased_csv_path = os.path.join(CSV_FOLDER, 'municipality-deceased.csv') old_hash = sha1sum(municipality_deceased_csv_path) with open(file=municipality_deceased_csv_path, encoding='utf-8') as f: rows = [row for row in csv.DictReader(f)] latest_date = str([val for val in df.index.values][-1]).split('T')[0] latest_date = datetime.strptime(latest_date, '%Y-%m-%d').date() while (date := datetime.strptime(rows[-1]['date'], '%Y-%m-%d').date()) < latest_date: rows.append(copy.deepcopy(rows[-1])) rows[-1]['date'] = str(date + timedelta(days=1)) # Write the rows collection back to the csv with open(municipality_deceased_csv_path, 'w', newline='', encoding='utf-8') as csvfile: writer = csv.DictWriter(csvfile, fieldnames=rows[0].keys()) writer.writeheader() for row in rows: writer.writerow(row) write_timestamp_file(filename=municipality_deceased_csv_path, old_hash=old_hash)
for key in [ 'kranj', 'ljubljana', 'domzale', 'saleske', 'koper', 'celje', 'maribor' ]: if key in components: return key raise Exception('No location key found: ', components) mapping = {} for column in df.columns: if column == 'NIB Measurements': mapping['NIB Measurements'] = 'measurements.nib' continue components = column.lower().split('_') location = get_location_key(components=components) components.remove(location) location = { 'saleske': 'velenje' }.get(location, location) # transform some of the location names mapping[column] = f'sewage.{location}.{"-".join(components)}' df.rename(mapper=mapping, axis='columns', inplace=True) # export to csv filename = os.path.join(CSV_FOLDER, 'sewage.csv') old_hash = sha1sum(filename) df.to_csv(filename) write_timestamp_file(filename=filename, old_hash=old_hash)
export_dataframe_to_csv(name='regions', dataframe=df.cumsum()) df = df.rolling(min_periods=1, window=14).sum().replace({0: None}).astype('Int64') \ .drop('region.n.neznano', axis='columns') export_dataframe_to_csv(name='active-regions', dataframe=df) with open(os.path.join(CSV_FOLDER, 'dict-municipality.csv')) as f: for row in csv.DictReader(f): municipalities[row['name'].lower()] = row if row['name_alt']: municipalities[row['name_alt'].lower()] = row # --- deceased-regions.csv --- # Copy paste latest row for every missing date deceased_regions_csv_path = os.path.join(CSV_FOLDER, 'deceased-regions.csv') old_hash = sha1sum(deceased_regions_csv_path) with open(deceased_regions_csv_path) as f: rows = [row for row in csv.DictReader(f)] latest_date = str([val for val in df.index.values][-1]).split('T')[0] latest_date = datetime.strptime(latest_date, '%Y-%m-%d').date() while (date := datetime.strptime(rows[-1]['date'], '%Y-%m-%d').date()) < latest_date: rows.append(copy.deepcopy(rows[-1])) rows[-1]['date'] = str(date + timedelta(days=1)) # Write the rows collection back to the csv with open(deceased_regions_csv_path, 'w', newline='') as csvfile: writer = csv.DictWriter(csvfile, fieldnames=rows[0].keys()) writer.writeheader() for row in rows: writer.writerow(row) write_timestamp_file(filename=deceased_regions_csv_path, old_hash=old_hash)
def do_process(data, settings): # Preparing directories utils.checkDir(utils.GET("object_dir"), "Object") if utils.GET("toposort_verbose_logging_dir") is not None and utils.GET( "toposort_verbose_logging_dir") != "": utils.checkDir(utils.GET("toposort_verbose_logging_dir"), "Toposort verbose logging") originalCXX = utils.GET("original_cxx_executable") originalCC = utils.GET("original_cc_executable") finishedList = Manager().list() totalLength = len(data["compile"]) compileTaskPool = Pool() console.log("Compiling .o (total: {})".format(totalLength)) for r in range(totalLength): i = data["compile"][r] execname = "(unknown)" cmdline = list(filter(lambda x: x != "", i.split(" "))) filehashpath = ["0" for i in range(0, 40)] for argnum in range(len(cmdline)): if cmdline[argnum] == originalCXX: cmdline[argnum] = utils.GET("targeted_cxx_executable") cmdline[argnum] += " -emit-llvm" elif cmdline[argnum] == originalCC: cmdline[argnum] = utils.GET("targeted_cc_executable") cmdline[argnum] += " -emit-llvm" elif cmdline[argnum] == "-o": filepath = realpath(cmdline[argnum + 1]) filehashpath = utils.sha1sum(filepath) sha1Table[filehashpath] = filepath cmdline[argnum + 1] = realpath( utils.GET("object_dir") + "/" + filehashpath) execname = utils.findName(filepath) elif cmdline[argnum] == "-c": cmdline[argnum] = "-S" elif cmdline[argnum] == "-g": cmdline[argnum] = "" command = " ".join(cmdline) compileTaskPool.apply_async(single_compile, args=(command, filehashpath, execname, r, totalLength, finishedList, settings.clean), error_callback=console_error_and_exit) compileTaskPool.close() compileTaskPool.join() # Construct the graph console.success("All object files are compiled.") console.info("Preparing linking relationships") graphData = data["scripts"] for i in graphData: itemPath = i["target"]["abs_path"] hashedItemPath = utils.sha1sum(itemPath) sha1Table[hashedItemPath] = itemPath itemDependencies = i["target"]["dependencies"] dependencyList[hashedItemPath] = utils.deduplicate( utils.pathToSha1(itemDependencies, sha1Table)) if hashedItemPath in dependencyList[hashedItemPath]: console.warn("Self-circle found. Ignoring.") dependencyList[hashedItemPath].remove(hashedItemPath) preserveProcess = utils.GET("preserve_process") if preserveProcess != None and preserveProcess != "": console.info("Saving metadata") sha1FilePath = utils.GET("object_dir") + "/" + preserveProcess try: json.dump( sha1Table, open(utils.GET("object_dir") + "/" + preserveProcess, "w")) console.success("Metadata saved.") except PermissionError: console.warn( "Process file {} is not writable, while preseve_process is on." .format(sha1FilePath)) console.info("Calculating linking sequence") try: currList = utils.topoSort(dependencyList, finishedList, sha1Table) except ValueError: console.error("Topo sort failed to complete. Please check your data.") sys.exit(1) console.success("Linking sequence calculated.") if settings.clean or settings.clean_linking: console.info("Cleaning linking targets") for i in dependencyList.keys(): if os.access(utils.GET("object_dir") + "/" + i, os.W_OK): os.unlink(utils.GET("object_dir") + "/" + i) console.success("Linking targets cleaned.") if len(currList) != len(graphData): console.warn("Bad consistance on linking recipe") console.debug("Linking sequence:", currList, "or", list(map(lambda x: sha1Table[x], currList))) console.info("Start linking") ctrLen = len(currList) p = Pool() for idx, obj in enumerate(currList): console.info("Linking {} ({}) [{}/{}]".format(sha1Table[obj], obj, idx + 1, ctrLen)) p.apply_async(single_linking, args=(obj, finishedList), error_callback=console_error_and_exit) p.close() p.join() console.success("All targets are linked.") console.success("Finished.")