def main(argv=sys.argv): if len(argv) < 2: usage(argv) config_uri = argv[1] options = parse_vars(argv[2:]) setup_logging(config_uri) settings = get_appsettings(config_uri, options=options) engine = engine_from_config(settings, 'sqlalchemy.') DBSession.configure(bind=engine) Base.metadata.drop_all(engine) Base.metadata.create_all(engine) with transaction.manager: user = User(username='******', password='******', email=settings.get('admin_mail'), activated=True, groups='admin') DBSession.add(user) DBSession.flush() for k, v in Settings.settings.iteritems(): if type(v) == dict: DBSession.add(Settings(key=unicode(k), value=unicode(v['value']))) else: if isinstance(v, basestring) and v.startswith('app_config.'): v = settings.get(v[len('app_config.'):]) DBSession.add(Settings(key=unicode(k), value=unicode(v))) DBSession.flush()
def setUp(self): from sqlalchemy import create_engine engine = create_engine('sqlite://') try: # nosetest vs pycharm test runner, lets flex settings = get_appsettings('../tests.ini') except: settings = get_appsettings('tests.ini') self.config = testing.setUp(settings=settings) DBSession.configure(bind=engine) Base.metadata.create_all(engine) with transaction.manager: self.founder = User(username='******', password='******', email='*****@*****.**', activated=True, active=True, firstname='founder1_firstname', lastname='founder1_lastname', city='founder1_city', state='founder1_state', title='founder1_title') self.company = BaseCompany(name='testcompany', founded=2015, description='company_desc', startup_teamdescription='company_team_desc', website='company_website', linkedin='company_linkedin', twitterhandle='company_twitter', angelcoprofile='company_angellist') self.admin = User(username='******', password='******', email=settings.get('admin_mail'), activated=False, groups='admin') DBSession.add(self.founder) DBSession.add(self.company) DBSession.add(self.admin) for k, v in Settings.settings.iteritems(): if type(v) == dict: DBSession.add(Settings(key=unicode(k), value=unicode(v['value']))) else: if isinstance(v, basestring) and v.startswith('app_config.'): v = settings.get(v[len('app_config.'):]) DBSession.add(Settings(key=unicode(k), value=unicode(v))) DBSession.flush() self.user_id = self.founder.user_id self.founders_created = []
def main(global_config, **settings): engine = engine_from_config(settings, 'sqlalchemy.') DBSession.configure(bind=engine) Base.metadata.bind = engine session_factory = session_factory_from_settings(settings) authn_policy = SessionAuthenticationPolicy(callback=groupfinder) authz_policy = ACLAuthorizationPolicy() config = Configurator(settings=settings, root_factory=RootFactory, authentication_policy=authn_policy, authorization_policy=authz_policy, session_factory=session_factory) config.include('pyramid_jinja2') add_jinja2_extension(config, ext.do) add_jinja2_extension(config, ext.loopcontrols) add_jinja2_extension(config, SelectiveHTMLCompress) add_jinja2_search_path(config, 'alchemist:general/layout/') add_jinja2_search_path(config, 'alchemist:general/blocks/') add_jinja2_search_path(config, 'alchemist:general/helpers/') config.include('pyramid_mailer') scheduler.add_jobstore('sqlalchemy', url=settings['sqlalchemy.url']) scheduler_tz = settings.get('scheduler.timezone') if scheduler_tz: scheduler.configure(timezone=scheduler_tz) scheduler.pyramid_registry = config.registry def get_user(request): id = authenticated_userid(request) if not id: return None return User.bid(id) config.add_request_method(get_user, 'user', reify=True) config.add_static_view('static', 'static', cache_max_age=3600) config.add_static_view('export-dumps', 'export-dumps', cache_max_age=3600) config.add_static_view('img', 'static/img', cache_max_age=3600) config.add_static_view('css', 'static/css', cache_max_age=3600) config.add_static_view('js', 'static/js', cache_max_age=3600) config.add_static_view('fonts', 'static/fonts', cache_max_age=3600) config.add_static_view('upload', 'upload', cache_max_age=3600) config.scan('alchemist') # it's important that this is after scan # TODO: make a venusian? decorator, maybe will not need the include then config.include('alchemist:qaforum.emails') scheduler.start() return config.make_wsgi_app()
def email_migrate(argv=sys.argv): if len(argv) < 2: usage(argv) config_uri = argv[1] options = parse_vars(argv[2:]) setup_logging(config_uri) settings = get_appsettings(config_uri, options=options) engine = engine_from_config(settings, 'sqlalchemy.') DBSession.configure(bind=engine) from petl import fromcsv conn_data = fromcsv('import_connections.csv').dicts() # conn_data = [] users_changed = [] users_added = 0 state_to_abbr = dict([(x[1], x[0]) for x in USPS_CHOICES]) print 'processing connections' with transaction.manager: for v in conn_data: user = DBSession.query(User).filter( or_(func.lower(User.email) == v['_email'].lower(), func.lower(User.linkedin_email) == v['_email'].lower(), and_(func.lower(User.firstname) == v['firstname'].lower(), func.lower(User.lastname) == v['lastname'].lower())) ).first() if user: for attrn, val in v.iteritems(): if not val or not attrn or attrn in ('_email',): continue val = val.strip() if attrn == 'company' and not user.company_id: repl_val = val.lower().rstrip('.').replace('llc', '').replace('corporation', ''). \ replace('corp', '').replace('inc', '').rstrip(',').strip() print 'searching:', repl_val, val company = DBSession.query(BaseCompany).filter( or_(BaseCompany.name == val, func.lower(BaseCompany.name) == val.lower(), func.regexp_replace(func.lower(func.btrim(BaseCompany.name)), ',\s(inc|corporation|corp|llc)\.?$', '') == repl_val, func.regexp_replace(func.lower(func.btrim(BaseCompany.name)), '\s(inc|corporation|corp|llc)\.?$', '') == repl_val )).first() if not company and val: company = BaseCompany(name=val) DBSession.add(company) val = company if attrn == 'area of expertise' and val: attrn = 'tag' tag = DBSession.query(Tag).filter(Tag.text == val).first() if tag and len(filter(lambda t: t.text == val, user.tags)) == 0: user.tags.append(tag) if attrn == 'state': val = state_to_abbr.get(val) if val and hasattr(user, attrn) and not getattr(user, attrn): print 'setting connection.%s.%s = %s' % (user.id, attrn, val) setattr(user, attrn, val) users_changed.append(user.id) DBSession.flush() users_changed = list(set(users_changed)) print '%s new users added, %s users changed: %s' % (users_added, len(users_changed), users_changed) del conn_data comp_data = fromcsv('companies_data.csv').dicts() companys_changed = [] companys_added = 0 print 'starting to process companies' with transaction.manager: for v in comp_data: if not v['name'].strip(): continue company = DBSession.query(BaseCompany).filter( or_(BaseCompany.name == v['name'].strip()) ).first() if company: for attrn, val in v.iteritems(): if not val or not attrn or attrn in ('_email',): continue val = val.strip() attrn = attrn.replace('company_', '') if attrn == 'founded': try: val = int(val) except: continue if val and hasattr(company, attrn) and not getattr(company, attrn): print 'setting company.%s.%s = %s' % (company.id, attrn, val) setattr(company, attrn, val) companys_changed.append(company.id) DBSession.flush() companys_changed = list(set(companys_changed)) print '%s new companys added, %s companys changed: %s' % ( companys_added, len(companys_changed), companys_changed)
def main(global_config, **settings): engine = engine_from_config(settings, 'sqlalchemy.') DBSession.configure(bind=engine) Base.metadata.bind = engine session_factory = session_factory_from_settings(settings) authn_policy = SessionAuthenticationPolicy(callback=groupfinder) authz_policy = ACLAuthorizationPolicy() config = Configurator(settings=settings, root_factory=RootFactory, authentication_policy=authn_policy, authorization_policy=authz_policy, session_factory=session_factory) config.include('pyramid_jinja2') add_jinja2_extension(config, ext.do) add_jinja2_extension(config, ext.loopcontrols) add_jinja2_extension(config, SelectiveHTMLCompress) add_jinja2_search_path(config, 'alchemist:templates') add_jinja2_search_path(config, 'alchemist:templates/layout') add_jinja2_search_path(config, 'alchemist:templates/helpers') add_jinja2_search_path(config, 'alchemist:templates/auth') add_jinja2_search_path(config, 'alchemist:templates/company') add_jinja2_search_path(config, 'alchemist:templates/blocks') add_jinja2_search_path(config, 'alchemist:templates/helpers') config.include('pyramid_dogpile_cache') cache = get_region('main') config.include('pyramid_mailer') def add_path(config, view, path, **kwargs): route_name = '%s_%s' % (qualname(view), path.replace('/', '_').split('{')[0]) config.add_route(route_name, path) config.add_view(view, route_name=route_name, **kwargs) config.add_directive('add_path', add_path) def get_user(request): id = authenticated_userid(request) if not id: return None return User.bid(id) config.add_request_method(get_user, 'user', reify=True) config.add_static_view('static', 'static', cache_max_age=3600) config.add_static_view('export-dumps', 'export-dumps', cache_max_age=3600) config.add_static_view('img', 'static/img', cache_max_age=3600) config.add_static_view('css', 'static/css', cache_max_age=3600) config.add_static_view('js', 'static/js', cache_max_age=3600) config.add_static_view('fonts', 'static/fonts', cache_max_age=3600) config.add_static_view('upload', 'upload', cache_max_age=3600) from alchemist import routes_admin from alchemist import routes_other from alchemist import routes_auth from alchemist import routes_user config.include(routes_admin.includeme) config.include(routes_other.includeme) config.include(routes_auth.includeme) config.include(routes_user.includeme) config.scan('alchemist.views.system') return config.make_wsgi_app()
def main(argv=sys.argv): if len(argv) < 2: usage(argv) config_uri = argv[1] options = parse_vars(argv[2:]) setup_logging(config_uri) settings = get_appsettings(config_uri, options=options) engine = engine_from_config(settings, 'sqlalchemy.') DBSession.configure(bind=engine) Base.metadata.drop_all(engine) Base.metadata.create_all(engine) with transaction.manager: user = User(username='******', password='******', email=settings.get('admin_mail'), activated=True, groups='admin') DBSession.add(user) DBSession.flush() for k, v in Settings.settings.iteritems(): if type(v) == dict: DBSession.add(Settings(key=unicode(k), value=unicode(v['value']))) else: if isinstance(v, basestring) and v.startswith('app_config.'): v = settings.get(v[len('app_config.'):]) DBSession.add(Settings(key=unicode(k), value=unicode(v))) DBSession.flush() print 'Starting to import data from spreadsheets' # if 1: if 0: from gdata.spreadsheet.service import SpreadsheetsService founder_columns = {} founders = [] to_founder_keys = {'Technical Co-Founder List?': 'technical', 'Email': 'email', 'Address': 'address', 'Alchemist Class #': 'alchemistclass', 'Public LinkedIn Profile': 'linkedin_profile', 'Areas of Expertise': 'tags', 'First Name': 'firstname', 'Last Name': 'lastname', 'Are you the CEO? (One and only one co-founder must be designated CEO)': 'ceo', 'Phone (Other)': '', 'Title': 'title', 'Are you an IoT company?': 'company.iot', 'Base Row': '', 'Company': 'company', 'Phone (Mobile)': 'phone', } to_company_keys = { 'name': 'name', 'description': 'description', 'twitterhandle': 'twitterhandle', 'email': 'email', 'website': 'website', 'tags': 'tags', 'braggingtext': 'braggingtext', 'totalraise': 'totalraise', 'haveraise': 'haveraise', 'logo': 'logo', 'alchemistclass': 'alchemistclass' } ss = SpreadsheetsService() ss.email = 'email' ss.password = '******' ss.ProgrammaticLogin() for shkey in []: print 'Starting to import companies from %s' % shkey linenr = 0 linestoskip = 7 headerline = 2 company_columns = {} for t in ss.GetListFeed(shkey).entry: linenr += 1 if headerline == linenr: for k, v in t.custom.iteritems(): if (v.text or '').strip() in to_company_keys: company_columns[k] = to_company_keys[v.text.strip()] print 'Got company_columns %s' % company_columns if linenr <= linestoskip: print 'skipping line %s' % linenr continue company_values = {} for k, v in t.custom.iteritems(): if k in company_columns: val = (getattr(v, 'text', '') or '').strip() if company_columns[k] in BaseCompany.boolean_fields: val = val.lower() == 'yes' company_values[company_columns[k]] = val company_values['tags'] = handle_if_tags('tags', company_values.get('tags', '').strip()) if not (company_values and company_values.get('name', '').strip() and company_values.get('description', '').strip()): print 'skipping because no values %s ' % company_values continue if 'logo' in company_values: logo_ = 'http%s' % company_values['logo'].split('http')[-1] if 'dropbox' in logo_: logo_ = logo_.split('?')[0] + '?raw=1' elif 'google' in logo_: logo_ = 'https://drive.google.com/uc?export=view&id=%s' % \ logo_.split('/')[-2] company_values['logo'] = logo_ company = DBSession.query(BaseCompany).filter( BaseCompany.name == company_values['name']).first() if company and company.description and company.email: print 'not importing %s already filled some info' % company.name continue elif company: for cf, cv in company_values.iteritems(): if hasattr(company, cf): setattr(company, cf, cv) else: DBSession.add(BaseCompany(**company_values)) DBSession.flush() print 'stating to import founders' for t in ss.GetListFeed('0As9IMrvjc_FqdGdiOHN3ZWhveWg5eHF1WVdmbGI4Vnc').entry: if not founder_columns: for k, v in t.custom.iteritems(): founder_columns[k] = v.text else: founder = {} for k, v in t.custom.iteritems(): if k in founder_columns and founder_columns[k] and \ to_founder_keys[founder_columns[k].strip()]: founder[to_founder_keys[founder_columns[k].strip()]] = v.text founders.append(founder) for founder in founders: company_name = founder.pop('company') is_iot = founder.pop('company.iot') if 'company.iot' in founder else False address = founder.pop('address') if 'address' in founder else '' try: if founder.get('email', '').strip(): assert re.match(r'[^@]+@[^@]+\.[^@]+', founder['email'].strip()), 'Invalid email address' except: print 'invalid email %s' % founder['email'] if 'email' in founder else '(no email)' founder['email'] = '*****@*****.**' if company_name: for c_ext in ['INC', 'Inc.', 'Inc', 'LLC', 'llc']: company_name = company_name.replace(c_ext, '').strip() company = DBSession.query(BaseCompany).filter( func.lower(BaseCompany.name) == func.lower(company_name)).first() for split_str in ['/', ' ', ' ']: if company: break for cn_split in company_name.split(split_str): company = DBSession.query(BaseCompany).filter( func.lower(BaseCompany.name).startswith(func.lower(cn_split.strip()))).first() if company: break if company: company_id = company.id if is_iot and is_iot.lower() == 'yes': company.iot = True c_address = getattr(company, 'address', '') if address and (not c_address or not str(c_address).strip()): company.address = address else: company = BaseCompany(name=company_name, iot=(is_iot and is_iot.lower() == 'yes'), address=address) DBSession.add(company) DBSession.flush() company_id = company.id for k in ['ceo', 'technical']: if k not in founder: founder[k] = False founder[k] = str(founder[k]).lower() == 'yes' if company and not company.alchemistclass and 'alchemistclass' in founder and founder[ 'alchemistclass'].strip(): try: company.alchemistclass = int(founder['alchemistclass'].strip()) except: pass if 'ceo' in founder and founder['ceo'] and not ('CEO' in founder['title'] or 'C.E.O' in founder['title'] or 'Chief Exec' in founder['title']): founder['title'] += (', ' if founder['title'].strip() else '') + 'CEO' if 'ceo' in founder and 'title' in founder and \ not founder['ceo'] and ('CEO' in founder['title'] or 'C.E.O' in founder['title'] or 'Chief Exec' in founder['title']): founder['ceo'] = True founder['company_id'] = company_id founder['activated'] = True founder['active'] = True founder['tags'] = handle_if_tags('tags', founder.get('tags', '').strip(), [',', 'and', ';']) DBSession.add(Founder(**founder))
def export_users_batch(argv=sys.argv): if len(argv) < 2: usage(argv) config_uri = argv[1] options = parse_vars(argv[2:]) setup_logging(config_uri) settings = get_appsettings(config_uri, options=options) engine = engine_from_config(settings, 'sqlalchemy.') DBSession.configure(bind=engine) create_attrs = {'name': 'nicename', 'email': 'email', 'contact_email': 'linkedin_email', 'phone': 'phone', #'user_hash': 'get_hmac', 'created_at': 'create_timestamp', 'poc': 'point_of_contact', 'primary_type': 'primary_type', 'total_sent_messages': 'sent', "total_connected_messages": 'connected', "conversion_messages": 'ratio', "Connect_Me-Accepted": 'connect_me_accept', "Connect_Me-Sent": 'connect_me_sent', "Connect_Me-Conv_Rate": 'connect_me_conv', "Request_Meeting-Accepted": 'request_meeting_accept', "Request_Meeting-Sent": 'request_meeting_sent', "Request_Meeting-Conv_Rate": 'request_meeting_conv', "Customer-To-Founder-Accepted": 'company_accept', "Customer-To-Founder-Sent": 'company_sent', "Customer-To-Founder-Conv_Rate": 'company_conv' } not_custom_attrs = ['name', 'email', 'created_at'] with transaction.manager: Intercom.app_id = Settings.get('intercom_id') Intercom.app_api_key = Settings.get('intercom_api_key') users = DBSession.query(User).filter(User.activated == True, User.login_enabled == True).all() print 'Rate limits', Intercom.rate_limit_details print 'Have %s users able to login, starting to update their info in intercom!' % len(users) for user in users: print 'processing user %s' % user.email try: intercom_user = IntercomUser.find(email=user.email) except: print 'created new user for intercom' intercom_user = IntercomUser.create(email=user.email) stats = user.get_connect_stats() changed = False for intercomattr, userattr in create_attrs.iteritems(): val = stats.get(userattr, getattr(user, userattr, None)) if hasattr(val, '__call__'): val = val() if val is None: continue changed = True if intercomattr in not_custom_attrs: setattr(intercom_user, intercomattr, val) else: intercom_user.custom_attributes[intercomattr] = val if user.company: intercom_user.companies = [{'id': user.company.id, 'name': user.company.name}] intercom_user.custom_attributes['class_number'] = user.company.alchemistclass \ if user.company else 'No class' if changed: print 'sending %s info' % user.email intercom_user.save() if Intercom.rate_limit_details.get('remaining', 0) <= 10: break