def explore(url, depth, good, bad, fn_remove, badv, dead, last_crawl):
#print url
pkgs = []
info = helper.open_dir(url)
if depth!=None and depth>0:
new_depth = depth - 1
elif depth==None:
new_depth = None
if info==None:
return []
for d,name,date in info:
if last_crawl!=None and date!=None and depth<2 and date<last_crawl:
continue
if d and name not in dead and (depth==None or depth>0):
if not name.endswith("/"):
name += "/"
pkgs += explore(url+name, new_depth, good, bad, fn_remove, badv, dead, last_crawl)
elif not d:
for token in fn_remove:
if token in name:
name = name.replace(token, "")
rel = parsers.parse_filename(name)
if rel!=None and ((good != None and rel.package in good and bad==None) or (good == None and bad != None and rel.package not in bad)) and not contains(rel.version, badv):
rel.released = date
#print "*",rel
pkgs.append(rel)
elif rel != None:
#print rel
pass
return pkgs
def get_releases(project_num, packages, bad_tokens, bad_versions, paths, last_crawl): rels = [] files = get_files(project_num, paths, last_crawl) for f in files: if contains(f[0],bad_tokens): continue rel = parsers.parse_filename(f[0]) if rel!=None and rel.package in packages and not contains(rel.version, bad_versions): rel.released = f[1] rels.append(rel) return rels
def get_releases(last_crawl=None): pkgs = [] for version in range(4,8): fn = "files/helper/php%d.txt"%version helper.open_url(MIRROR+"/releases/index.php?serialize=1&version=%d&max=3000"%version,fn) f = open(fn) s = f.read() f.close() d = deserialize(s) f = flatten(d) if f == None: #no filenames found continue for date,fn in f: rel = parsers.parse_filename(fn) if rel!=None and "pl" not in rel.version: rel.released = date pkgs.append(rel) return pkgs
def explore(url, depth, good, bad, fn_remove, badv, dead, last_crawl):
pkgs = []
print url
info = helper.open_dir(url)
if depth != None and depth > 0:
new_depth = depth - 1
elif depth == None:
new_depth = None
if info == None:
return []
for d, name, date in info:
if last_crawl != None and date != None and depth < 2 and date < last_crawl:
continue
if d and name not in dead and (depth == None or depth > 0):
if not name.endswith("/"):
name += "/"
pkgs += explore(url + name, new_depth, good, bad, fn_remove, badv,
dead, last_crawl)
elif not d:
for token in fn_remove:
if token in name:
name = name.replace(token, "")
rel = parsers.parse_filename(name)
if rel != None and (
(good != None and rel.package in good and bad == None) or
(good == None and bad != None and rel.package
not in bad)) and not contains(rel.version, badv):
rel.released = date
#print "*",rel
pkgs.append(rel)
elif rel != None:
#print rel
pass
return pkgs
def get_releases(last_crawl=None):
pkgs = []
for version in range(4, 8):
fn = "files/helper/php%d.txt" % version
helper.open_url(
MIRROR +
"/releases/index.php?serialize=1&version=%d&max=3000" % version,
fn)
f = open(fn)
s = f.read()
f.close()
d = deserialize(s)
f = flatten(d)
if f == None: #no filenames found
continue
for date, fn in f:
rel = parsers.parse_filename(fn)
if rel != None and "pl" not in rel.version:
rel.released = date
pkgs.append(rel)
return pkgs
