def parse_rar_archive(filename, sha256sum=False):
print("Creating infolist for %s" % filename)
with rarfile.RarFile(filename, 'r') as afile:
infolist = afile.infolist()
print("Infolist created for %s" % filename)
tmpdir = tempfile.mkdtemp(prefix='extracted-rar-')
cmd = ['rar', 'x', filename, tmpdir]
subprocess.check_call(cmd)
here = path.cwd()
os.chdir(tmpdir)
td = path(tmpdir)
print("Parsing info")
entries = list()
for ainfo in infolist:
if ainfo.isdir():
print("Skipping directory %s" % ainfo.filename)
continue
parsed = parse_archive_info(ainfo, 'rar')
parsed['bytesize'] = ainfo.file_size
size, cksum = get_extracted_info(td, ainfo, sha256sum=sha256sum)
if size != ainfo.file_size:
raise RuntimeError("Sizes don't match %s" % ainfo.filename)
parsed['sha256sum'] = cksum
entries.append(parsed)
cmd = ['rm', '-fr', tmpdir]
subprocess.check_call(cmd)
os.chdir(here)
return entries
def parse_rar_archive(filename, sha256sum=False):
print("Creating infolist for %s" % filename)
with rarfile.RarFile(filename, 'r') as afile:
infolist = afile.infolist()
print("Infolist created for %s" % filename)
tmpdir = tempfile.mkdtemp(prefix='extracted-rar-')
cmd = ['rar', 'x', filename, tmpdir]
subprocess.check_call(cmd)
here = path.cwd()
os.chdir(tmpdir)
td = path(tmpdir)
print("Parsing info")
entries = list()
for ainfo in infolist:
if ainfo.isdir():
print("Skipping directory %s" % ainfo.filename)
continue
parsed = parse_archive_info(ainfo, 'rar')
parsed['bytesize'] = ainfo.file_size
size, cksum = get_extracted_info(td, ainfo, sha256sum=sha256sum)
if size != ainfo.file_size:
raise RuntimeError("Sizes don't match %s" % ainfo.filename)
parsed['sha256sum'] = cksum
entries.append(parsed)
cmd = ['rm', '-fr', tmpdir]
subprocess.check_call(cmd)
os.chdir(here)
return entries
def get_files(directory):
here = path.cwd()
directory = path(directory)
os.chdir(directory)
walker = directory.walk(filter=FILES)
files = [p.relative() for p in walker if not p.relative().startswith(".")]
os.chdir(here)
return files
def get_extracted_info(parent_dir, fileinfo, sha256sum=False):
parent_dir = path(parent_dir)
filename = fileinfo.filename
try:
extracted_name = path(parent_dir, filename)
except UnicodeEncodeError:
print("UnicodeEncodeError with %s" % filename)
filename = filename.encode('utf8')
extracted_name = path(parent_dir, filename)
# make sure extracted_name exists
if not extracted_name.isfile():
raise RuntimeError("%s is not a file" % extracted_name)
ifile = open(extracted_name)
ifile.seek(0, 2)
file_size = ifile.tell()
ifile.seek(0)
cksum = None
if sha256sum:
cksum = get_sha256sum(ifile)
return file_size, cksum
def get_extracted_info(parent_dir, fileinfo, sha256sum=False):
parent_dir = path(parent_dir)
filename = fileinfo.filename
try:
extracted_name = path(parent_dir, filename)
except UnicodeEncodeError:
print("UnicodeEncodeError with %s" % filename)
filename = filename.encode('utf8')
extracted_name = path(parent_dir, filename)
# make sure extracted_name exists
if not extracted_name.isfile():
raise RuntimeError("%s is not a file" % extracted_name)
ifile = open(extracted_name)
ifile.seek(0, 2)
file_size = ifile.tell()
ifile.seek(0)
cksum = None
if sha256sum:
cksum = get_sha256sum(ifile)
return file_size, cksum
def force_single_instance(self):
if len(sys.argv) >= 1:
app = path(sys.argv[0]).stem
if self.islock():
msg = 'An instance of %s is already running ' % app + \
'(lock file: %s).' % self.getlockfile()
self.errorlog(msg)
sys.exit(1)
atexit.register(self.unlock)
self.lock()
def getapplogger(name=None, debug=False):
logger = None
if not name:
logger = _getlogger()
else:
logger = _getlogger('laworks.' + name)
formatter = _getformatter()
logfile = path("/var/log/laworks/")
if not logfile.exists():
logfile.mkdir()
if debug:
logger.setLevel(logging.DEBUG)
loghandler = logging.FileHandler("/var/log/laworks/" + name, 'a')
loghandler.setFormatter(formatter)
logger.addHandler(loghandler)
return logger
def __init__(self, debug=False, master=True, root=True, haslog=False):
""" Initialize Class variables. Extend as needed """
if root:
self.check_run_by_root()
self.name = path(sys.argv[0]).absolute().stem
if master:
self.check_run_on_master()
self.args = sys.argv
self.version = LPTVERSION
usage = self.name + " help"
self.parser = OptionParser(usage)
if haslog:
self.logger = log.getapplogger(self.name, debug)
self.log = self.logger.info
self.warnlog = self.logger.warn
self.errlog = self.logger.fatal
def assert_git_directory(directory):
directory = path(directory)
assert directory.isdir()
cmd = ['git', '-C', directory, 'rev-parse']
subprocess.check_call(cmd)
add_file_to_git(filename, clone_directory)
else:
add_file_to_annex(filename)
def main(directory):
if not os.path.isdir(".git/annex/"):
raise RuntimeError, "Run this from the annex toplevel"
files = list(get_files(directory))
print "%d files" % len(files)
for filename in files:
add_file_to_repo(filename, directory)
if filename.endswith(".md"):
# print "Checking", filename
if markdown_has_papers(filename, directory):
# print "%s should have papers" % filename
parse_markdown(filename, directory)
if __name__ == "__main__":
pwl_local_directory = path("/tmp/papers-we-love")
if not os.path.isdir(pwl_local_directory):
repo = "https://github.com/papers-we-love/papers-we-love.git"
cmd = ["git", "clone", repo, pwl_local_directory]
subprocess.check_call(cmd)
if not os.path.isdir(pwl_local_directory):
raise RuntimeError, "bad clone"
cmd = ["git", "-C", "/tmp/papers-we-love", "pull"]
subprocess.check_call(cmd)
main(pwl_local_directory)
def _getappname(self):
"""Returns the full path to the current program"""
return path('/var/lock/') + self.name
def __init__(self, repo_path):
self.repo_path = path(repo_path)
if line.startswith('define'):
has_define = True
continue
if line.strip() and not line.startswith(' '):
raise RuntimeError, "Bad line in %s" % filename
else:
if not line.strip():
newlines.append('\n')
else:
nl = line[2:]
if nl.startswith('module.exports'):
nl = nl[7:]
newlines.append(nl)
if not has_define:
raise RuntimeError, "%s has no define" % filename
return newlines
for pathobj in path('.').walk():
if os.path.isfile(pathobj) and pathobj.endswith('.coffee'):
newlines = check_file(pathobj)
#print 66*"+"
#print pathobj
#print 66*"+"
#print ''.join(newlines)
with file(pathobj, 'w') as outfile:
for line in newlines:
outfile.write(line)
def set_repo_path(self, repo_path):
repo_path = path(repo_path)
if not repo_path.isdir():
raise RuntimeError, "%s doesn't exist." % repo_path
self.repo_path = repo_path