def main(argv=sys.argv):
if len(argv) < 2:
usage(argv)
config_uri = argv[1]
options = parse_vars(argv[2:])
setup_logging(config_uri)
settings = get_appsettings(config_uri, options=options)
engine = engine_from_config(settings, 'sqlalchemy.')
DBSession.configure(bind=engine)
Base.metadata.drop_all(engine)
Base.metadata.create_all(engine)
with transaction.manager:
user = User(username='******', password='******', email=settings.get('admin_mail'), activated=True,
groups='admin')
DBSession.add(user)
DBSession.flush()
for k, v in Settings.settings.iteritems():
if type(v) == dict:
DBSession.add(Settings(key=unicode(k), value=unicode(v['value'])))
else:
if isinstance(v, basestring) and v.startswith('app_config.'):
v = settings.get(v[len('app_config.'):])
DBSession.add(Settings(key=unicode(k), value=unicode(v)))
DBSession.flush()
def setUp(self):
from sqlalchemy import create_engine
engine = create_engine('sqlite://')
try:
# nosetest vs pycharm test runner, lets flex
settings = get_appsettings('../tests.ini')
except:
settings = get_appsettings('tests.ini')
self.config = testing.setUp(settings=settings)
DBSession.configure(bind=engine)
Base.metadata.create_all(engine)
with transaction.manager:
self.founder = User(username='******', password='******', email='*****@*****.**', activated=True,
active=True, firstname='founder1_firstname', lastname='founder1_lastname',
city='founder1_city', state='founder1_state', title='founder1_title')
self.company = BaseCompany(name='testcompany', founded=2015, description='company_desc',
startup_teamdescription='company_team_desc', website='company_website',
linkedin='company_linkedin', twitterhandle='company_twitter',
angelcoprofile='company_angellist')
self.admin = User(username='******', password='******', email=settings.get('admin_mail'), activated=False,
groups='admin')
DBSession.add(self.founder)
DBSession.add(self.company)
DBSession.add(self.admin)
for k, v in Settings.settings.iteritems():
if type(v) == dict:
DBSession.add(Settings(key=unicode(k), value=unicode(v['value'])))
else:
if isinstance(v, basestring) and v.startswith('app_config.'):
v = settings.get(v[len('app_config.'):])
DBSession.add(Settings(key=unicode(k), value=unicode(v)))
DBSession.flush()
self.user_id = self.founder.user_id
self.founders_created = []
def main(global_config, **settings):
engine = engine_from_config(settings, 'sqlalchemy.')
DBSession.configure(bind=engine)
Base.metadata.bind = engine
session_factory = session_factory_from_settings(settings)
authn_policy = SessionAuthenticationPolicy(callback=groupfinder)
authz_policy = ACLAuthorizationPolicy()
config = Configurator(settings=settings,
root_factory=RootFactory,
authentication_policy=authn_policy,
authorization_policy=authz_policy,
session_factory=session_factory)
config.include('pyramid_jinja2')
add_jinja2_extension(config, ext.do)
add_jinja2_extension(config, ext.loopcontrols)
add_jinja2_extension(config, SelectiveHTMLCompress)
add_jinja2_search_path(config, 'alchemist:general/layout/')
add_jinja2_search_path(config, 'alchemist:general/blocks/')
add_jinja2_search_path(config, 'alchemist:general/helpers/')
config.include('pyramid_mailer')
scheduler.add_jobstore('sqlalchemy', url=settings['sqlalchemy.url'])
scheduler_tz = settings.get('scheduler.timezone')
if scheduler_tz:
scheduler.configure(timezone=scheduler_tz)
scheduler.pyramid_registry = config.registry
def get_user(request):
id = authenticated_userid(request)
if not id:
return None
return User.bid(id)
config.add_request_method(get_user, 'user', reify=True)
config.add_static_view('static', 'static', cache_max_age=3600)
config.add_static_view('export-dumps', 'export-dumps', cache_max_age=3600)
config.add_static_view('img', 'static/img', cache_max_age=3600)
config.add_static_view('css', 'static/css', cache_max_age=3600)
config.add_static_view('js', 'static/js', cache_max_age=3600)
config.add_static_view('fonts', 'static/fonts', cache_max_age=3600)
config.add_static_view('upload', 'upload', cache_max_age=3600)
config.scan('alchemist')
# it's important that this is after scan
# TODO: make a venusian? decorator, maybe will not need the include then
config.include('alchemist:qaforum.emails')
scheduler.start()
return config.make_wsgi_app()
def email_migrate(argv=sys.argv):
if len(argv) < 2:
usage(argv)
config_uri = argv[1]
options = parse_vars(argv[2:])
setup_logging(config_uri)
settings = get_appsettings(config_uri, options=options)
engine = engine_from_config(settings, 'sqlalchemy.')
DBSession.configure(bind=engine)
from petl import fromcsv
conn_data = fromcsv('import_connections.csv').dicts()
# conn_data = []
users_changed = []
users_added = 0
state_to_abbr = dict([(x[1], x[0]) for x in USPS_CHOICES])
print 'processing connections'
with transaction.manager:
for v in conn_data:
user = DBSession.query(User).filter(
or_(func.lower(User.email) == v['_email'].lower(),
func.lower(User.linkedin_email) == v['_email'].lower(),
and_(func.lower(User.firstname) == v['firstname'].lower(),
func.lower(User.lastname) == v['lastname'].lower()))
).first()
if user:
for attrn, val in v.iteritems():
if not val or not attrn or attrn in ('_email',):
continue
val = val.strip()
if attrn == 'company' and not user.company_id:
repl_val = val.lower().rstrip('.').replace('llc', '').replace('corporation', ''). \
replace('corp', '').replace('inc', '').rstrip(',').strip()
print 'searching:', repl_val, val
company = DBSession.query(BaseCompany).filter(
or_(BaseCompany.name == val,
func.lower(BaseCompany.name) == val.lower(),
func.regexp_replace(func.lower(func.btrim(BaseCompany.name)),
',\s(inc|corporation|corp|llc)\.?$', '') == repl_val,
func.regexp_replace(func.lower(func.btrim(BaseCompany.name)),
'\s(inc|corporation|corp|llc)\.?$', '') == repl_val
)).first()
if not company and val:
company = BaseCompany(name=val)
DBSession.add(company)
val = company
if attrn == 'area of expertise' and val:
attrn = 'tag'
tag = DBSession.query(Tag).filter(Tag.text == val).first()
if tag and len(filter(lambda t: t.text == val, user.tags)) == 0:
user.tags.append(tag)
if attrn == 'state':
val = state_to_abbr.get(val)
if val and hasattr(user, attrn) and not getattr(user, attrn):
print 'setting connection.%s.%s = %s' % (user.id, attrn, val)
setattr(user, attrn, val)
users_changed.append(user.id)
DBSession.flush()
users_changed = list(set(users_changed))
print '%s new users added, %s users changed: %s' % (users_added, len(users_changed), users_changed)
del conn_data
comp_data = fromcsv('companies_data.csv').dicts()
companys_changed = []
companys_added = 0
print 'starting to process companies'
with transaction.manager:
for v in comp_data:
if not v['name'].strip():
continue
company = DBSession.query(BaseCompany).filter(
or_(BaseCompany.name == v['name'].strip())
).first()
if company:
for attrn, val in v.iteritems():
if not val or not attrn or attrn in ('_email',):
continue
val = val.strip()
attrn = attrn.replace('company_', '')
if attrn == 'founded':
try:
val = int(val)
except:
continue
if val and hasattr(company, attrn) and not getattr(company, attrn):
print 'setting company.%s.%s = %s' % (company.id, attrn, val)
setattr(company, attrn, val)
companys_changed.append(company.id)
DBSession.flush()
companys_changed = list(set(companys_changed))
print '%s new companys added, %s companys changed: %s' % (
companys_added, len(companys_changed), companys_changed)
def main(global_config, **settings):
engine = engine_from_config(settings, 'sqlalchemy.')
DBSession.configure(bind=engine)
Base.metadata.bind = engine
session_factory = session_factory_from_settings(settings)
authn_policy = SessionAuthenticationPolicy(callback=groupfinder)
authz_policy = ACLAuthorizationPolicy()
config = Configurator(settings=settings,
root_factory=RootFactory,
authentication_policy=authn_policy,
authorization_policy=authz_policy,
session_factory=session_factory)
config.include('pyramid_jinja2')
add_jinja2_extension(config, ext.do)
add_jinja2_extension(config, ext.loopcontrols)
add_jinja2_extension(config, SelectiveHTMLCompress)
add_jinja2_search_path(config, 'alchemist:templates')
add_jinja2_search_path(config, 'alchemist:templates/layout')
add_jinja2_search_path(config, 'alchemist:templates/helpers')
add_jinja2_search_path(config, 'alchemist:templates/auth')
add_jinja2_search_path(config, 'alchemist:templates/company')
add_jinja2_search_path(config, 'alchemist:templates/blocks')
add_jinja2_search_path(config, 'alchemist:templates/helpers')
config.include('pyramid_dogpile_cache')
cache = get_region('main')
config.include('pyramid_mailer')
def add_path(config, view, path, **kwargs):
route_name = '%s_%s' % (qualname(view), path.replace('/', '_').split('{')[0])
config.add_route(route_name, path)
config.add_view(view, route_name=route_name, **kwargs)
config.add_directive('add_path', add_path)
def get_user(request):
id = authenticated_userid(request)
if not id:
return None
return User.bid(id)
config.add_request_method(get_user, 'user', reify=True)
config.add_static_view('static', 'static', cache_max_age=3600)
config.add_static_view('export-dumps', 'export-dumps', cache_max_age=3600)
config.add_static_view('img', 'static/img', cache_max_age=3600)
config.add_static_view('css', 'static/css', cache_max_age=3600)
config.add_static_view('js', 'static/js', cache_max_age=3600)
config.add_static_view('fonts', 'static/fonts', cache_max_age=3600)
config.add_static_view('upload', 'upload', cache_max_age=3600)
from alchemist import routes_admin
from alchemist import routes_other
from alchemist import routes_auth
from alchemist import routes_user
config.include(routes_admin.includeme)
config.include(routes_other.includeme)
config.include(routes_auth.includeme)
config.include(routes_user.includeme)
config.scan('alchemist.views.system')
return config.make_wsgi_app()
def main(argv=sys.argv):
if len(argv) < 2:
usage(argv)
config_uri = argv[1]
options = parse_vars(argv[2:])
setup_logging(config_uri)
settings = get_appsettings(config_uri, options=options)
engine = engine_from_config(settings, 'sqlalchemy.')
DBSession.configure(bind=engine)
Base.metadata.drop_all(engine)
Base.metadata.create_all(engine)
with transaction.manager:
user = User(username='******', password='******', email=settings.get('admin_mail'), activated=True,
groups='admin')
DBSession.add(user)
DBSession.flush()
for k, v in Settings.settings.iteritems():
if type(v) == dict:
DBSession.add(Settings(key=unicode(k), value=unicode(v['value'])))
else:
if isinstance(v, basestring) and v.startswith('app_config.'):
v = settings.get(v[len('app_config.'):])
DBSession.add(Settings(key=unicode(k), value=unicode(v)))
DBSession.flush()
print 'Starting to import data from spreadsheets'
# if 1:
if 0:
from gdata.spreadsheet.service import SpreadsheetsService
founder_columns = {}
founders = []
to_founder_keys = {'Technical Co-Founder List?': 'technical',
'Email': 'email',
'Address': 'address',
'Alchemist Class #': 'alchemistclass',
'Public LinkedIn Profile': 'linkedin_profile',
'Areas of Expertise': 'tags',
'First Name': 'firstname',
'Last Name': 'lastname',
'Are you the CEO? (One and only one co-founder must be designated CEO)': 'ceo',
'Phone (Other)': '',
'Title': 'title',
'Are you an IoT company?': 'company.iot',
'Base Row': '',
'Company': 'company',
'Phone (Mobile)': 'phone',
}
to_company_keys = {
'name': 'name',
'description': 'description',
'twitterhandle': 'twitterhandle',
'email': 'email',
'website': 'website',
'tags': 'tags',
'braggingtext': 'braggingtext',
'totalraise': 'totalraise',
'haveraise': 'haveraise',
'logo': 'logo',
'alchemistclass': 'alchemistclass'
}
ss = SpreadsheetsService()
ss.email = 'email'
ss.password = '******'
ss.ProgrammaticLogin()
for shkey in []:
print 'Starting to import companies from %s' % shkey
linenr = 0
linestoskip = 7
headerline = 2
company_columns = {}
for t in ss.GetListFeed(shkey).entry:
linenr += 1
if headerline == linenr:
for k, v in t.custom.iteritems():
if (v.text or '').strip() in to_company_keys:
company_columns[k] = to_company_keys[v.text.strip()]
print 'Got company_columns %s' % company_columns
if linenr <= linestoskip:
print 'skipping line %s' % linenr
continue
company_values = {}
for k, v in t.custom.iteritems():
if k in company_columns:
val = (getattr(v, 'text', '') or '').strip()
if company_columns[k] in BaseCompany.boolean_fields:
val = val.lower() == 'yes'
company_values[company_columns[k]] = val
company_values['tags'] = handle_if_tags('tags', company_values.get('tags', '').strip())
if not (company_values and company_values.get('name', '').strip() and
company_values.get('description', '').strip()):
print 'skipping because no values %s ' % company_values
continue
if 'logo' in company_values:
logo_ = 'http%s' % company_values['logo'].split('http')[-1]
if 'dropbox' in logo_:
logo_ = logo_.split('?')[0] + '?raw=1'
elif 'google' in logo_:
logo_ = 'https://drive.google.com/uc?export=view&id=%s' % \
logo_.split('/')[-2]
company_values['logo'] = logo_
company = DBSession.query(BaseCompany).filter(
BaseCompany.name == company_values['name']).first()
if company and company.description and company.email:
print 'not importing %s already filled some info' % company.name
continue
elif company:
for cf, cv in company_values.iteritems():
if hasattr(company, cf):
setattr(company, cf, cv)
else:
DBSession.add(BaseCompany(**company_values))
DBSession.flush()
print 'stating to import founders'
for t in ss.GetListFeed('0As9IMrvjc_FqdGdiOHN3ZWhveWg5eHF1WVdmbGI4Vnc').entry:
if not founder_columns:
for k, v in t.custom.iteritems():
founder_columns[k] = v.text
else:
founder = {}
for k, v in t.custom.iteritems():
if k in founder_columns and founder_columns[k] and \
to_founder_keys[founder_columns[k].strip()]:
founder[to_founder_keys[founder_columns[k].strip()]] = v.text
founders.append(founder)
for founder in founders:
company_name = founder.pop('company')
is_iot = founder.pop('company.iot') if 'company.iot' in founder else False
address = founder.pop('address') if 'address' in founder else ''
try:
if founder.get('email', '').strip():
assert re.match(r'[^@]+@[^@]+\.[^@]+', founder['email'].strip()), 'Invalid email address'
except:
print 'invalid email %s' % founder['email'] if 'email' in founder else '(no email)'
founder['email'] = '*****@*****.**'
if company_name:
for c_ext in ['INC', 'Inc.', 'Inc', 'LLC', 'llc']:
company_name = company_name.replace(c_ext, '').strip()
company = DBSession.query(BaseCompany).filter(
func.lower(BaseCompany.name) == func.lower(company_name)).first()
for split_str in ['/', ' ', ' ']:
if company:
break
for cn_split in company_name.split(split_str):
company = DBSession.query(BaseCompany).filter(
func.lower(BaseCompany.name).startswith(func.lower(cn_split.strip()))).first()
if company:
break
if company:
company_id = company.id
if is_iot and is_iot.lower() == 'yes':
company.iot = True
c_address = getattr(company, 'address', '')
if address and (not c_address or not str(c_address).strip()):
company.address = address
else:
company = BaseCompany(name=company_name, iot=(is_iot and is_iot.lower() == 'yes'),
address=address)
DBSession.add(company)
DBSession.flush()
company_id = company.id
for k in ['ceo', 'technical']:
if k not in founder:
founder[k] = False
founder[k] = str(founder[k]).lower() == 'yes'
if company and not company.alchemistclass and 'alchemistclass' in founder and founder[
'alchemistclass'].strip():
try:
company.alchemistclass = int(founder['alchemistclass'].strip())
except:
pass
if 'ceo' in founder and founder['ceo'] and not ('CEO' in founder['title'] or 'C.E.O' in founder['title']
or 'Chief Exec' in founder['title']):
founder['title'] += (', ' if founder['title'].strip() else '') + 'CEO'
if 'ceo' in founder and 'title' in founder and \
not founder['ceo'] and ('CEO' in founder['title'] or 'C.E.O' in founder['title']
or 'Chief Exec' in founder['title']):
founder['ceo'] = True
founder['company_id'] = company_id
founder['activated'] = True
founder['active'] = True
founder['tags'] = handle_if_tags('tags', founder.get('tags', '').strip(), [',', 'and', ';'])
DBSession.add(Founder(**founder))
def export_users_batch(argv=sys.argv):
if len(argv) < 2:
usage(argv)
config_uri = argv[1]
options = parse_vars(argv[2:])
setup_logging(config_uri)
settings = get_appsettings(config_uri, options=options)
engine = engine_from_config(settings, 'sqlalchemy.')
DBSession.configure(bind=engine)
create_attrs = {'name': 'nicename',
'email': 'email',
'contact_email': 'linkedin_email',
'phone': 'phone',
#'user_hash': 'get_hmac',
'created_at': 'create_timestamp',
'poc': 'point_of_contact',
'primary_type': 'primary_type',
'total_sent_messages': 'sent',
"total_connected_messages": 'connected',
"conversion_messages": 'ratio',
"Connect_Me-Accepted": 'connect_me_accept',
"Connect_Me-Sent": 'connect_me_sent',
"Connect_Me-Conv_Rate": 'connect_me_conv',
"Request_Meeting-Accepted": 'request_meeting_accept',
"Request_Meeting-Sent": 'request_meeting_sent',
"Request_Meeting-Conv_Rate": 'request_meeting_conv',
"Customer-To-Founder-Accepted": 'company_accept',
"Customer-To-Founder-Sent": 'company_sent',
"Customer-To-Founder-Conv_Rate": 'company_conv'
}
not_custom_attrs = ['name', 'email', 'created_at']
with transaction.manager:
Intercom.app_id = Settings.get('intercom_id')
Intercom.app_api_key = Settings.get('intercom_api_key')
users = DBSession.query(User).filter(User.activated == True, User.login_enabled == True).all()
print 'Rate limits', Intercom.rate_limit_details
print 'Have %s users able to login, starting to update their info in intercom!' % len(users)
for user in users:
print 'processing user %s' % user.email
try:
intercom_user = IntercomUser.find(email=user.email)
except:
print 'created new user for intercom'
intercom_user = IntercomUser.create(email=user.email)
stats = user.get_connect_stats()
changed = False
for intercomattr, userattr in create_attrs.iteritems():
val = stats.get(userattr, getattr(user, userattr, None))
if hasattr(val, '__call__'):
val = val()
if val is None:
continue
changed = True
if intercomattr in not_custom_attrs:
setattr(intercom_user, intercomattr, val)
else:
intercom_user.custom_attributes[intercomattr] = val
if user.company:
intercom_user.companies = [{'id': user.company.id,
'name': user.company.name}]
intercom_user.custom_attributes['class_number'] = user.company.alchemistclass \
if user.company else 'No class'
if changed:
print 'sending %s info' % user.email
intercom_user.save()
if Intercom.rate_limit_details.get('remaining', 0) <= 10:
break
