def check_link(pif, link, rejects=[], visible=False):
if link:
print(link, visible)
link = pif.dbh.depref('link_line', link)
lstatus = 'unset'
if visible and (link['flags'] & config.FLAG_LINK_LINE_HIDDEN or link['page_id'] == 'links.rejects'):
return
print(link['id'], link['url'],)
if link['flags'] & config.FLAG_LINK_LINE_NOT_VERIFIABLE or link['link_type'] in 'tfpn':
lstatus = 'NoVer'
elif link['link_type'] in 'bglsx':
# ret = is_blacklisted(link['url'], rejects)
# if ret:
# print(link['id'], link['section_id'], link['url'], "BLACKLISTED", ret)
# pif.dbh.dbi.remove('link_line', 'id=%s' % link['id'])
lurl = link['url']
if lurl.startswith('/'):
lurl = 'https://www.bamca.org' + lurl
try:
url = urllib.request.urlopen(urllib.request.Request(
lurl, headers={
'User-Agent':
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.10; rv:42.0) Gecko/20100101 Firefox/42.0'}))
lstatus = 'H' + str(url.code)
except HTTPError as c:
print('http error:', c.code)
lstatus = 'H' + str(c.code)
except URLError as c:
print('url error:', c.reason)
lstatus = 'U' + str(c.reason[0])
except Exception:
lstatus = 'exc'
print(lstatus)
if link.get('last_status') != lstatus:
pif.dbh.update_link_line({'id': str(link['id']), 'last_status': lstatus})
def unhide_makes(pif, *args):
# if any casting for a make has a section that is shown, unhide the make
makes = args if args else [
x['vehicle_make.id'] for x in pif.dbh.fetch_vehicle_makes()
]
# count_shown_q = '''
# select count(*) from casting,casting_make,section where
# casting.section_id=section.id and section.page_id='manno' and
# casting_make.casting_id=casting.id and casting_make.make_id='%s'
# '''
# res = pif.dbh.raw_execute(count_shown_q)
for make_id in makes:
res = pif.dbh.fetch(
'casting,casting_make,section',
columns='count(*) as c',
one=True,
where=
"casting.section_id=section.id and section.page_id='manno' and "
"casting_make.casting_id=casting.id and casting_make.make_id='%s'"
% make_id)
make = pif.dbh.depref('vehicle_make',
pif.dbh.fetch_vehicle_make(make_id))
flag = 0 if res['c'] else config.FLAG_ITEM_HIDDEN
print(make, flag)
pif.dbh.update_vehicle_make(make['id'], {'flags': flag})
def dump_database(pif):
cols = [
'id', 'pic', 'box_size', 'year', 'additional_text', 'bottom', 'sides',
'end_flap', 'model_name', 'notes'
]
titles = {
'id': 'id',
'mod_id': 'mod_id',
'box_type': 'typ',
'pic_id': 'p',
'pic': 'pic',
'box_size': 'z',
'year': 'year',
'additional_text': 'addl_text',
'bottom': 'bottom',
'sides': 'sides',
'end_flap': 'end_flap',
'model_name': 'model_name',
'notes': 'notes',
}
db = pif.dbh.depref('box_type', pif.dbh.fetch('box_type'))
lens = {col: 0 for col in cols}
for row in db:
row['pic'] = '%s-%s%s' % (row['mod_id'], row['box_type'][0],
row['pic_id'])
for col in cols[1:]:
lens[col] = max(lens[col], len(row[col]))
lens['id'] = 4
# id | mod_id | typ | p | z | year | addl_text | bottom | sides | end_flap | model_name | notes
print(' | '.join([('%%-%ds' % lens[col]) % titles[col] for col in cols]))
for row in db:
print(' | '.join([('%%-%ds' % lens[col]) % row[col]
for col in cols]).strip())
def check_var_vs_categories(pif):
wheres = [
"variation_select.mod_id=variation.mod_id",
"variation_select.var_id=variation.var"
]
vars = {}
for vvs in pif.dbh.fetch('variation_select,variation',
where=wheres,
tag='VarVS'):
key = (
vvs['variation.mod_id'],
vvs['variation.var'],
)
vars.setdefault(key, dict())
vars[key]['vc'] = set(vvs['variation.category'].split(
' ')) if vvs['variation.category'] else set()
vars[key].setdefault('vsc', set())
vars[key]['vsc'].add(vvs['variation_select.category'])
vl = [
key + (
' '.join(sorted(vars[key]['vc'])),
' '.join(sorted(vars[key]['vsc'])),
) for key in sorted(vars.keys()) if vars[key]['vc'] - vars[key]['vsc']
]
fmt = pif.dbh.preformat_results(vl)
for v in vl:
print(fmt % v)
def call_main(switches="",
options="",
long_options={},
version="",
short_help="",
long_help="",
envar=None,
noerror=False,
defaults={},
doglob=False):
try:
switch, filelist = get_command_line(switches=switches,
options=options,
long_options=long_options,
version=version,
short_help=short_help,
long_help=long_help,
envar=envar,
noerror=noerror,
defaults=defaults,
doglob=doglob)
ret = main_fn(switch, filelist)
useful.write_comment()
if ret:
print(ret)
except SystemExit:
pass
def check_castings(pif):
# pif.form.set_val('section', 'all')
sec_ids = [
x['section.id']
for x in pif.dbh.fetch_sections(where={'page_id': pif.page_id})
]
totals = {}
tags = []
for sec in sec_ids:
pif.form.set_val('section', sec)
manf = mannum.MannoFile(pif, withaliases=True)
llineup = manf.run_picture_list(pif)
for ent in llineup['section'][0]['range'][0]['entry']:
pass
disp = ['%-8s' % sec]
for tot in llineup['totals']:
if tot['tag'] not in tags:
tags.append(tot['tag'])
totals[tot['tag']] = [0, 0]
totals[tot['tag']][0] += tot['have']
totals[tot['tag']][1] += tot['total']
disp.extend(['%7d ' % tot['have'], '%7d ' % tot['total']])
print(''.join(disp))
disp = ['totals ']
for tag in tags:
disp.extend(['%7d ' % totals[tag][0], '%7d ' % totals[tag][1]])
print(''.join(disp))
def call_main(page_id='cli',
form_key='',
defval='',
args='',
dbedit=None,
switches='',
options=''):
useful.header_done(False)
pif = None
try:
switch, filelist = get_command_line(switches, options)
for f in filelist:
if f.startswith('page_id='):
page_id = f[8:]
if isinstance(page_id, pifile.PageInfoFile):
pif = page_id
else:
pif = get_page_info(page_id, form_key, defval, args, dbedit)
pif.switch, pif.filelist = switch, filelist
ret = main_fn(pif)
useful.write_comment()
if ret:
print(ret)
except SystemExit:
pass
except useful.SimpleError as e:
print('***', e.value)
def simple_html(status=404):
if not useful.is_header_done():
print('Content-Type: text/html\n\n')
print('Status:', status, http.client.responses.get(status, ''))
# print('<!--\n' + str(os.environ) + '-->')
useful.header_done()
useful.write_comment()
def read_presets(pdir):
if os.path.exists(os.path.join(pdir, '.ima')):
presets = eval(open(os.path.join(pdir, '.ima')).read())
print('read_presets:', presets, '<br>')
presets['save'] = [1]
return presets
return dict()
def get_years(pif, region, ystart, yend, pr_count, im_count):
for year in range(ystart, yend + 1):
count = get_year(pif, region, year)
pr_count += count[0]
im_count += count[1]
print(" %s %s %-4d / %-4d" % (year, region, count[1], count[0]))
return pr_count, im_count
def count_tables(pif):
for tab in pif.dbh.dbi.execute('show tables')[0]:
cnts = pif.dbh.dbi.execute('select count(*) from ' + tab[0])
print("%7d %s" % (cnts[0][0][0], tab[0]))
if pif.switch['v']:
cols = pif.dbh.dbi.execute('desc ' + tab[0])
for col in cols[0]:
print(' ', col)
def char_test(fidet):
pass
noch = []
if fidet:
for ch in fidet:
if ch not in vrdata.ok_letters:
noch.append(ord(ch))
if noch:
print('<br>', noch)
def check_lib_man(pif):
man_ids = set(
[x.lower().replace('/', '_') for x in pif.dbh.fetch_casting_ids()])
man_lib = set(os.listdir('.' + config.LIB_MAN_DIR))
print("id's without libraries:")
print(' '.join(sorted(man_ids - man_lib)))
print()
print("libraries without id's:")
print(' '.join(sorted(man_lib - man_ids)))
def check_blacklisted_links(pif, sections=None):
reject, banned = links.read_blacklist(pif)
pif.dbh.set_verbose(True)
for sec in sections if sections else [None]:
for link in pif.dbh.fetch_link_lines(section=sec):
link = pif.dbh.depref('link_line', link)
if link['link_type'] in 'blsxg':
ret = is_blacklisted(link['url'], reject)
if ret:
print(link['id'], link['section_id'], link['url'], "BLACKLISTED", ret)
def cars_main(pif):
pif.render.print_html()
db = CarsFile(
useful.relpath(config.SRC_DIR,
pif.form.get_str('page', 'cars') + '.dat'))
print(pif.render.format_head())
render_cars(pif, db)
print(pif.render.format_tail())
def import_psdc(pif):
pref = 'https://www.publicsafetydiecast.com/'
u = requests.get('https://www.publicsafetydiecast.com/Matchbox_MAN.htm').text
u_re = re.compile(r'<a href="(?P<u>[^"]*)".*?<font.*?>(?P<i>.*?)<\/font>')
q = get_links(pif, 'PSDC')
ul = list(set([x['link_line.url'] for x in q]))
pl = list(set(u_re.findall(u)))
for ln in pl:
if not pref + ln[0] in ul:
print(ln[1], pref + ln[0])
def header_done(is_web=True, silent=False):
global _format_web, _pending_comments
_format_web = is_web
is_header_done(True)
ostr = '\n'.join([format_string(*x) for x in _pending_comments])
_pending_comments = list()
if silent:
return ostr
if ostr:
print(ostr)
def read_column_change(fil):
changes = dict()
for ln in fil:
try:
mnl, col, colto = ln.split('|')
except ValueError:
print('ValueError:', ln, '<br>')
continue
for mn in mnl.split(';'):
changes.setdefault(mn, list())
changes[mn].append([col.split(';'), colto.split(';')])
return changes
def check_lineup_model(pif):
print('lineup_model')
res = pif.dbh.raw_execute(
'''select page_id, region, number, count(*) from lineup_model group by page_id, region, number'''
)
check_q(res[0])
return
problems = set()
for row in res[0]:
if row[3] > 1:
problems.add(row[:2])
print(problems if problems else 'all ok')
def check_detail(pif):
print('detail')
res = pif.dbh.raw_execute(
'''select mod_id, var_id, attr_id, count(*) from detail group by mod_id, var_id, attr_id'''
)
check_q(res[0])
return
problems = set()
for row in res[0]:
if row[3] > 1:
problems.add(row[:2])
print(problems)
def check_man_mappings(pif, sections):
for section in sections:
mans = pif.dbh.fetch_casting_list(section_id=section, page_id='manno')
mans.sort(key=lambda x: x['casting.id'])
for man in mans:
cid = man['casting.id']
aliases = pif.dbh.fetch_aliases(cid, 'mack')
mack_nums = single.get_mack_numbers(pif, cid,
man['base_id.model_type'],
aliases)
if not mack_nums:
print(cid)
def update_links(pif):
links = pif.dbh.fetch_link_lines()
good_ids = [x for x in range(100, 3000)]
bad_ids = []
for lnk in links:
id = lnk['link_line.id']
if id in good_ids:
good_ids.remove(id)
elif id < 100 and not lnk['link_line.flags'] & 64:
bad_ids.append(id)
bad_ids.sort()
for ids in zip(good_ids, bad_ids):
print("update link_line set id=%d where id=%d;" % ids)
def read_attr_change(fil):
changes = dict()
for ln in fil:
try:
cols, detfr, detto = ln.split('|')
except ValueError:
print('ValueError:', ln, '<br>')
continue
for col in cols.split(';'):
changes.setdefault(col, list())
for det in detfr.split(';'):
changes[col].append([det, detto])
return changes
def write_php_config_file(pif):
print("Writing PHP config file.")
fin = open('../bin/config.py').readlines()
fout = ['<?php\n', '// Generated file. Do not modify.\n']
for ln in fin:
if ln.startswith('#'):
ln = '//' + ln[1:]
elif ln.find('=') >= 0:
ln = '$' + ln.replace('\n', ';\n')
fout.append(ln)
fout.append('?>\n')
open('../htdocs/config.php', 'w').writelines(fout)
print()
def check_attribute_pictures(pif, *filelist):
fl = os.listdir('.' + config.IMG_DIR_ADD)
print(sorted(list(set([x[0] for x in fl if x.endswith('.jpg')]))))
for pref in filelist:
print(pref)
pics = pif.dbh.fetch_attribute_pictures_by_type(pref)
for pic in pics:
if pic['attribute_picture.picture_id']:
pic_name = '{}_{}-{}.jpg'.format(
pic['attribute_picture.attr_type'],
pic['attribute_picture.mod_id'].lower(),
pic['attribute_picture.picture_id'])
else:
pic_name = '{}_{}.jpg'.format(
pic['attribute_picture.attr_type'],
pic['attribute_picture.mod_id'].lower())
if pic_name in fl:
fl.remove(pic_name)
else:
print('no pic for', pic_name)
for fn in fl:
if fn.startswith(pref + '_') and fn.endswith('.jpg'):
fn = fn[2:-4]
mod_id, pic_id = fn.split('-', 1) if '-' in fn else [fn, '']
rec = {
'mod_id': mod_id,
'attr_id': 0,
'attr_type': pref,
'picture_id': pic_id,
'description': ''
}
print(rec)
def check_tables(pif, *filelist):
if filelist:
for tab in filelist:
check_table(pif, tab)
print()
else:
# found_ch = set()
tabs = pif.dbh.dbi.execute('show tables')
for tab in tabs[0]:
tab = tab[0]
if tab == 'counter':
continue
check_table(pif, tab)
print()
def check_images(pif):
global casting_ids, variation_ids
casting_ids = [x['base_id.id'].lower() for x in pif.dbh.fetch_base_ids()]
variation_ids = [
x['variation.mod_id'].lower() + '-' + x['variation.var'].lower()
for x in pif.dbh.fetch_variations_bare()
]
for key in sorted(checks.keys()):
if checks[key]:
print(key)
checks[key](pif, key)
print()
def check_man(pif, dn):
files = glob.glob('.' + dn + '/*.*')
files.sort()
c = 0
for fn in files:
try:
root, ext = os.path.splitext(os.path.basename(fn))
if not ext or ext[1:] not in imglib.itypes:
continue
root = root.lower()
var = ''
if '-' in root:
root, var = root.rsplit('-', 1)
if len(root) > 1 and root[1] == '_':
root = root[2:]
if root not in casting_ids:
print(fn, "missing base", root)
elif var and (root + '-' + var) not in variation_ids:
print(fn, "missing var", root, var)
else:
c += 1
except Exception as e:
print(fn, "fail -", e)
raise
print(c, 'ok')
def grab_list(ll, fl):
for url in fl:
fn = url[url.rfind('/') + 1:]
libdir = useful.relpath('.', config.LIB_MAN_DIR,
ll['link_line.page_id'][7:].lower())
if not os.path.exists(libdir):
errors.append((ll, url))
sfn = os.path.join(libdir, fn)
dot = sfn.rfind('.')
sfn = sfn[:dot] + sfn[dot:].lower()
if os.path.exists(sfn):
print(sfn, 'already exists')
else:
img = grab_page(ll['pth'] + '/' + url)
save(ll, sfn, img)
def do_var_masses(pif, tform):
for fn, var in pif.form.get_list(start='var.'):
print('<hr>')
print(fn, var, '<br>')
eform = images.EditForm(pif, tdir=pif.render.pic_dir, fn=fn)
eform.ot = 'jpg'
eform.tysz = 's'
eform.read_file('')
eform.man = eform.calc_man()
eform.var = eform.nvar = var
eform.mass_resize(pif)
var_id = pif.form.get('msspromote')
if var_id:
mod_id = eform.calc_man()
imglib.promote_picture(pif, mod_id, var_id)
