def status(self): path = self.__status_file() if os.path.exists(path): return slurp(path) name = os.path.basename(self.dir) i = name.find('.') if i == -1: return 'Done' return name[i + 1:].capitalize()
def status(self): path = self.__status_file() if os.path.exists(path): return slurp(path) name = os.path.basename(self.dir) i = name.find('.') if i == -1: return 'Done' return name[i+1:].capitalize()
def __init__(self, dir): self.__base_url = 'http://download.xcsoar.org/mapgen/data/' self.__cmd_7zip = '7zr' self.__cmd_wget = 'wget' self.__dir = os.path.abspath(dir) self.__manifest = None if not os.path.exists(self.__dir): os.makedirs(self.__dir) subprocess.check_call([self.__cmd_wget, '-q', '-N', '-P', self.__dir, self.__base_url + 'checksums']) self.__checksums = {} for line in slurp(os.path.join(self.__dir, 'checksums')).split("\n"): line = line.strip() if line != '': line = line.split(None, 1) self.__checksums[line[1]] = line[0]
def get_next(dir_jobs): if not os.path.exists(dir_jobs): return None next_dir = None next_ts = time.time() for entry in os.listdir(dir_jobs): dir = os.path.join(dir_jobs, entry) # Only directories can be jobs if not os.path.isdir(dir): continue ts = None try: ts = float(slurp(os.path.join(dir, 'timestamp'))) except Exception as e: print("Could not read timestamp file for job {}\n{}".format( dir, e)) continue age = time.time() - ts # Check if there is a running job which is expired if (dir.endswith('.locked') or dir.endswith('.working')) and age > 60 * 60: print('Delete expired job {}'.format(dir)) shutil.rmtree(dir) continue # Find an enqueued job if dir.endswith('.queued'): if ts < next_ts: next_dir = dir next_ts = ts elif age > 24 * 7 * 60 * 60: # Delete if download is expired print('Delete expired job {}'.format(dir)) shutil.rmtree(dir) if next_dir != None: job = Job(next_dir) job.__move('.working') return job return None
def __init__(self, dir): self.__base_url = 'http://mapgen-data.sigkill.ch/' self.__cmd_7zip = '7zr' self.__cmd_wget = 'wget' self.__dir = os.path.abspath(dir) self.__manifest = None if not os.path.exists(self.__dir): os.makedirs(self.__dir) subprocess.check_call([ self.__cmd_wget, '-q', '-N', '-P', self.__dir, self.__base_url + 'checksums' ]) self.__checksums = {} for line in slurp(os.path.join(self.__dir, 'checksums')).split("\n"): line = line.strip() if line != '': line = line.split(None, 1) self.__checksums[line[1]] = line[0]
def __get_local_checksum(self, file): md5_path = file + '.md5' if os.path.exists(md5_path): return slurp(md5_path) if not os.path.isfile(file): return None md5 = hashlib.md5() file = open(file, 'rb') try: while True: data = file.read(0xFFFF) if not data: break md5.update(data) finally: file.close() md5 = md5.hexdigest() spew(md5_path, md5) return md5
def get_next(dir_jobs): if not os.path.exists(dir_jobs): return None next_dir = None next_ts = time.time() for entry in os.listdir(dir_jobs): dir = os.path.join(dir_jobs, entry) # Only directories can be jobs if not os.path.isdir(dir): continue ts = None try: ts = float(slurp(os.path.join(dir, 'timestamp'))) except Exception as e: print("Could not read timestamp file for job {}\n{}".format(dir, e)) continue age = time.time() - ts # Check if there is a running job which is expired if (dir.endswith('.locked') or dir.endswith('.working')) and age > 60*60: print('Delete expired job {}'.format(dir)) shutil.rmtree(dir) continue # Find an enqueued job if dir.endswith('.queued'): if ts < next_ts: next_dir = dir next_ts = ts elif age > 24*7*60*60: # Delete if download is expired print('Delete expired job {}'.format(dir)) shutil.rmtree(dir) if next_dir != None: job = Job(next_dir) job.__move('.working') return job return None
def manifest(self): if not self.__manifest: self.__manifest = json.loads(slurp(self.retrieve('manifest'))) return self.__manifest