def check_link(pif, link, rejects=[], visible=False): if link: print(link, visible) link = pif.dbh.depref('link_line', link) lstatus = 'unset' if visible and (link['flags'] & config.FLAG_LINK_LINE_HIDDEN or link['page_id'] == 'links.rejects'): return print(link['id'], link['url'],) if link['flags'] & config.FLAG_LINK_LINE_NOT_VERIFIABLE or link['link_type'] in 'tfpn': lstatus = 'NoVer' elif link['link_type'] in 'bglsx': # ret = is_blacklisted(link['url'], rejects) # if ret: # print(link['id'], link['section_id'], link['url'], "BLACKLISTED", ret) # pif.dbh.dbi.remove('link_line', 'id=%s' % link['id']) lurl = link['url'] if lurl.startswith('/'): lurl = 'https://www.bamca.org' + lurl try: url = urllib.request.urlopen(urllib.request.Request( lurl, headers={ 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.10; rv:42.0) Gecko/20100101 Firefox/42.0'})) lstatus = 'H' + str(url.code) except HTTPError as c: print('http error:', c.code) lstatus = 'H' + str(c.code) except URLError as c: print('url error:', c.reason) lstatus = 'U' + str(c.reason[0]) except Exception: lstatus = 'exc' print(lstatus) if link.get('last_status') != lstatus: pif.dbh.update_link_line({'id': str(link['id']), 'last_status': lstatus})
def unhide_makes(pif, *args): # if any casting for a make has a section that is shown, unhide the make makes = args if args else [ x['vehicle_make.id'] for x in pif.dbh.fetch_vehicle_makes() ] # count_shown_q = ''' # select count(*) from casting,casting_make,section where # casting.section_id=section.id and section.page_id='manno' and # casting_make.casting_id=casting.id and casting_make.make_id='%s' # ''' # res = pif.dbh.raw_execute(count_shown_q) for make_id in makes: res = pif.dbh.fetch( 'casting,casting_make,section', columns='count(*) as c', one=True, where= "casting.section_id=section.id and section.page_id='manno' and " "casting_make.casting_id=casting.id and casting_make.make_id='%s'" % make_id) make = pif.dbh.depref('vehicle_make', pif.dbh.fetch_vehicle_make(make_id)) flag = 0 if res['c'] else config.FLAG_ITEM_HIDDEN print(make, flag) pif.dbh.update_vehicle_make(make['id'], {'flags': flag})
def dump_database(pif): cols = [ 'id', 'pic', 'box_size', 'year', 'additional_text', 'bottom', 'sides', 'end_flap', 'model_name', 'notes' ] titles = { 'id': 'id', 'mod_id': 'mod_id', 'box_type': 'typ', 'pic_id': 'p', 'pic': 'pic', 'box_size': 'z', 'year': 'year', 'additional_text': 'addl_text', 'bottom': 'bottom', 'sides': 'sides', 'end_flap': 'end_flap', 'model_name': 'model_name', 'notes': 'notes', } db = pif.dbh.depref('box_type', pif.dbh.fetch('box_type')) lens = {col: 0 for col in cols} for row in db: row['pic'] = '%s-%s%s' % (row['mod_id'], row['box_type'][0], row['pic_id']) for col in cols[1:]: lens[col] = max(lens[col], len(row[col])) lens['id'] = 4 # id | mod_id | typ | p | z | year | addl_text | bottom | sides | end_flap | model_name | notes print(' | '.join([('%%-%ds' % lens[col]) % titles[col] for col in cols])) for row in db: print(' | '.join([('%%-%ds' % lens[col]) % row[col] for col in cols]).strip())
def check_var_vs_categories(pif): wheres = [ "variation_select.mod_id=variation.mod_id", "variation_select.var_id=variation.var" ] vars = {} for vvs in pif.dbh.fetch('variation_select,variation', where=wheres, tag='VarVS'): key = ( vvs['variation.mod_id'], vvs['variation.var'], ) vars.setdefault(key, dict()) vars[key]['vc'] = set(vvs['variation.category'].split( ' ')) if vvs['variation.category'] else set() vars[key].setdefault('vsc', set()) vars[key]['vsc'].add(vvs['variation_select.category']) vl = [ key + ( ' '.join(sorted(vars[key]['vc'])), ' '.join(sorted(vars[key]['vsc'])), ) for key in sorted(vars.keys()) if vars[key]['vc'] - vars[key]['vsc'] ] fmt = pif.dbh.preformat_results(vl) for v in vl: print(fmt % v)
def call_main(switches="", options="", long_options={}, version="", short_help="", long_help="", envar=None, noerror=False, defaults={}, doglob=False): try: switch, filelist = get_command_line(switches=switches, options=options, long_options=long_options, version=version, short_help=short_help, long_help=long_help, envar=envar, noerror=noerror, defaults=defaults, doglob=doglob) ret = main_fn(switch, filelist) useful.write_comment() if ret: print(ret) except SystemExit: pass
def check_castings(pif): # pif.form.set_val('section', 'all') sec_ids = [ x['section.id'] for x in pif.dbh.fetch_sections(where={'page_id': pif.page_id}) ] totals = {} tags = [] for sec in sec_ids: pif.form.set_val('section', sec) manf = mannum.MannoFile(pif, withaliases=True) llineup = manf.run_picture_list(pif) for ent in llineup['section'][0]['range'][0]['entry']: pass disp = ['%-8s' % sec] for tot in llineup['totals']: if tot['tag'] not in tags: tags.append(tot['tag']) totals[tot['tag']] = [0, 0] totals[tot['tag']][0] += tot['have'] totals[tot['tag']][1] += tot['total'] disp.extend(['%7d ' % tot['have'], '%7d ' % tot['total']]) print(''.join(disp)) disp = ['totals '] for tag in tags: disp.extend(['%7d ' % totals[tag][0], '%7d ' % totals[tag][1]]) print(''.join(disp))
def call_main(page_id='cli', form_key='', defval='', args='', dbedit=None, switches='', options=''): useful.header_done(False) pif = None try: switch, filelist = get_command_line(switches, options) for f in filelist: if f.startswith('page_id='): page_id = f[8:] if isinstance(page_id, pifile.PageInfoFile): pif = page_id else: pif = get_page_info(page_id, form_key, defval, args, dbedit) pif.switch, pif.filelist = switch, filelist ret = main_fn(pif) useful.write_comment() if ret: print(ret) except SystemExit: pass except useful.SimpleError as e: print('***', e.value)
def simple_html(status=404): if not useful.is_header_done(): print('Content-Type: text/html\n\n') print('Status:', status, http.client.responses.get(status, '')) # print('<!--\n' + str(os.environ) + '-->') useful.header_done() useful.write_comment()
def read_presets(pdir): if os.path.exists(os.path.join(pdir, '.ima')): presets = eval(open(os.path.join(pdir, '.ima')).read()) print('read_presets:', presets, '<br>') presets['save'] = [1] return presets return dict()
def get_years(pif, region, ystart, yend, pr_count, im_count): for year in range(ystart, yend + 1): count = get_year(pif, region, year) pr_count += count[0] im_count += count[1] print(" %s %s %-4d / %-4d" % (year, region, count[1], count[0])) return pr_count, im_count
def count_tables(pif): for tab in pif.dbh.dbi.execute('show tables')[0]: cnts = pif.dbh.dbi.execute('select count(*) from ' + tab[0]) print("%7d %s" % (cnts[0][0][0], tab[0])) if pif.switch['v']: cols = pif.dbh.dbi.execute('desc ' + tab[0]) for col in cols[0]: print(' ', col)
def char_test(fidet): pass noch = [] if fidet: for ch in fidet: if ch not in vrdata.ok_letters: noch.append(ord(ch)) if noch: print('<br>', noch)
def check_lib_man(pif): man_ids = set( [x.lower().replace('/', '_') for x in pif.dbh.fetch_casting_ids()]) man_lib = set(os.listdir('.' + config.LIB_MAN_DIR)) print("id's without libraries:") print(' '.join(sorted(man_ids - man_lib))) print() print("libraries without id's:") print(' '.join(sorted(man_lib - man_ids)))
def check_blacklisted_links(pif, sections=None): reject, banned = links.read_blacklist(pif) pif.dbh.set_verbose(True) for sec in sections if sections else [None]: for link in pif.dbh.fetch_link_lines(section=sec): link = pif.dbh.depref('link_line', link) if link['link_type'] in 'blsxg': ret = is_blacklisted(link['url'], reject) if ret: print(link['id'], link['section_id'], link['url'], "BLACKLISTED", ret)
def cars_main(pif): pif.render.print_html() db = CarsFile( useful.relpath(config.SRC_DIR, pif.form.get_str('page', 'cars') + '.dat')) print(pif.render.format_head()) render_cars(pif, db) print(pif.render.format_tail())
def import_psdc(pif): pref = 'https://www.publicsafetydiecast.com/' u = requests.get('https://www.publicsafetydiecast.com/Matchbox_MAN.htm').text u_re = re.compile(r'<a href="(?P<u>[^"]*)".*?<font.*?>(?P<i>.*?)<\/font>') q = get_links(pif, 'PSDC') ul = list(set([x['link_line.url'] for x in q])) pl = list(set(u_re.findall(u))) for ln in pl: if not pref + ln[0] in ul: print(ln[1], pref + ln[0])
def header_done(is_web=True, silent=False): global _format_web, _pending_comments _format_web = is_web is_header_done(True) ostr = '\n'.join([format_string(*x) for x in _pending_comments]) _pending_comments = list() if silent: return ostr if ostr: print(ostr)
def read_column_change(fil): changes = dict() for ln in fil: try: mnl, col, colto = ln.split('|') except ValueError: print('ValueError:', ln, '<br>') continue for mn in mnl.split(';'): changes.setdefault(mn, list()) changes[mn].append([col.split(';'), colto.split(';')]) return changes
def check_lineup_model(pif): print('lineup_model') res = pif.dbh.raw_execute( '''select page_id, region, number, count(*) from lineup_model group by page_id, region, number''' ) check_q(res[0]) return problems = set() for row in res[0]: if row[3] > 1: problems.add(row[:2]) print(problems if problems else 'all ok')
def check_detail(pif): print('detail') res = pif.dbh.raw_execute( '''select mod_id, var_id, attr_id, count(*) from detail group by mod_id, var_id, attr_id''' ) check_q(res[0]) return problems = set() for row in res[0]: if row[3] > 1: problems.add(row[:2]) print(problems)
def check_man_mappings(pif, sections): for section in sections: mans = pif.dbh.fetch_casting_list(section_id=section, page_id='manno') mans.sort(key=lambda x: x['casting.id']) for man in mans: cid = man['casting.id'] aliases = pif.dbh.fetch_aliases(cid, 'mack') mack_nums = single.get_mack_numbers(pif, cid, man['base_id.model_type'], aliases) if not mack_nums: print(cid)
def update_links(pif): links = pif.dbh.fetch_link_lines() good_ids = [x for x in range(100, 3000)] bad_ids = [] for lnk in links: id = lnk['link_line.id'] if id in good_ids: good_ids.remove(id) elif id < 100 and not lnk['link_line.flags'] & 64: bad_ids.append(id) bad_ids.sort() for ids in zip(good_ids, bad_ids): print("update link_line set id=%d where id=%d;" % ids)
def read_attr_change(fil): changes = dict() for ln in fil: try: cols, detfr, detto = ln.split('|') except ValueError: print('ValueError:', ln, '<br>') continue for col in cols.split(';'): changes.setdefault(col, list()) for det in detfr.split(';'): changes[col].append([det, detto]) return changes
def write_php_config_file(pif): print("Writing PHP config file.") fin = open('../bin/config.py').readlines() fout = ['<?php\n', '// Generated file. Do not modify.\n'] for ln in fin: if ln.startswith('#'): ln = '//' + ln[1:] elif ln.find('=') >= 0: ln = '$' + ln.replace('\n', ';\n') fout.append(ln) fout.append('?>\n') open('../htdocs/config.php', 'w').writelines(fout) print()
def check_attribute_pictures(pif, *filelist): fl = os.listdir('.' + config.IMG_DIR_ADD) print(sorted(list(set([x[0] for x in fl if x.endswith('.jpg')])))) for pref in filelist: print(pref) pics = pif.dbh.fetch_attribute_pictures_by_type(pref) for pic in pics: if pic['attribute_picture.picture_id']: pic_name = '{}_{}-{}.jpg'.format( pic['attribute_picture.attr_type'], pic['attribute_picture.mod_id'].lower(), pic['attribute_picture.picture_id']) else: pic_name = '{}_{}.jpg'.format( pic['attribute_picture.attr_type'], pic['attribute_picture.mod_id'].lower()) if pic_name in fl: fl.remove(pic_name) else: print('no pic for', pic_name) for fn in fl: if fn.startswith(pref + '_') and fn.endswith('.jpg'): fn = fn[2:-4] mod_id, pic_id = fn.split('-', 1) if '-' in fn else [fn, ''] rec = { 'mod_id': mod_id, 'attr_id': 0, 'attr_type': pref, 'picture_id': pic_id, 'description': '' } print(rec)
def check_tables(pif, *filelist): if filelist: for tab in filelist: check_table(pif, tab) print() else: # found_ch = set() tabs = pif.dbh.dbi.execute('show tables') for tab in tabs[0]: tab = tab[0] if tab == 'counter': continue check_table(pif, tab) print()
def check_images(pif): global casting_ids, variation_ids casting_ids = [x['base_id.id'].lower() for x in pif.dbh.fetch_base_ids()] variation_ids = [ x['variation.mod_id'].lower() + '-' + x['variation.var'].lower() for x in pif.dbh.fetch_variations_bare() ] for key in sorted(checks.keys()): if checks[key]: print(key) checks[key](pif, key) print()
def check_man(pif, dn): files = glob.glob('.' + dn + '/*.*') files.sort() c = 0 for fn in files: try: root, ext = os.path.splitext(os.path.basename(fn)) if not ext or ext[1:] not in imglib.itypes: continue root = root.lower() var = '' if '-' in root: root, var = root.rsplit('-', 1) if len(root) > 1 and root[1] == '_': root = root[2:] if root not in casting_ids: print(fn, "missing base", root) elif var and (root + '-' + var) not in variation_ids: print(fn, "missing var", root, var) else: c += 1 except Exception as e: print(fn, "fail -", e) raise print(c, 'ok')
def grab_list(ll, fl): for url in fl: fn = url[url.rfind('/') + 1:] libdir = useful.relpath('.', config.LIB_MAN_DIR, ll['link_line.page_id'][7:].lower()) if not os.path.exists(libdir): errors.append((ll, url)) sfn = os.path.join(libdir, fn) dot = sfn.rfind('.') sfn = sfn[:dot] + sfn[dot:].lower() if os.path.exists(sfn): print(sfn, 'already exists') else: img = grab_page(ll['pth'] + '/' + url) save(ll, sfn, img)
def do_var_masses(pif, tform): for fn, var in pif.form.get_list(start='var.'): print('<hr>') print(fn, var, '<br>') eform = images.EditForm(pif, tdir=pif.render.pic_dir, fn=fn) eform.ot = 'jpg' eform.tysz = 's' eform.read_file('') eform.man = eform.calc_man() eform.var = eform.nvar = var eform.mass_resize(pif) var_id = pif.form.get('msspromote') if var_id: mod_id = eform.calc_man() imglib.promote_picture(pif, mod_id, var_id)